From 50bbafd58c15fac596a29a5b53fe6d68f96cc4ae Mon Sep 17 00:00:00 2001 From: Octavian-Mihai Matei Date: Mon, 23 Mar 2026 14:04:52 +0100 Subject: [PATCH 001/113] CONSOLE: eos rm two or more files at once. Fixes EOS-4147 --- console/commands/com_proto_rm.cc | 78 ++++++++++++++++++++++++++++---- test/eos-instance-test | 39 ++++++++++++++++ 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/console/commands/com_proto_rm.cc b/console/commands/com_proto_rm.cc index 04db9d2610..49d5cd2f77 100644 --- a/console/commands/com_proto_rm.cc +++ b/console/commands/com_proto_rm.cc @@ -68,7 +68,7 @@ RmHelper::ParseCommand(const char* arg) eos::console::RmProto* rm = mReq.mutable_rm(); eos::common::StringTokenizer tokenizer(arg); bool noconfirmation = false; - + tokenizer.GetLine(); while ((option = tokenizer.GetToken(false)).length() > 0 && @@ -130,7 +130,8 @@ RmHelper::ParseCommand(const char* arg) eos::common::Path cPath(path.c_str()); if (path.length()) { - mNeedsConfirmation = rm->recursive() && (cPath.GetSubPathSize() < 4) && !noconfirmation; + mNeedsConfirmation = + rm->recursive() && (cPath.GetSubPathSize() < 4) && !noconfirmation; } return true; @@ -146,22 +147,81 @@ int com_protorm(char* arg) global_retc = EINVAL; return EINVAL; } + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); - RmHelper rm(gGlobalOpts); + std::vector paths; + std::string optStr; + std::string currentPath; + bool inOptions = true; + + XrdOucString token; + while ((token = tokenizer.GetToken(false)).length() > 0) { + if (inOptions && token.beginswith("-")) { + if (!optStr.empty()) { + optStr += " "; + } + optStr += token.c_str(); + } else { + inOptions = false; + + // Check if this is a new path + if (token.beginswith("/") || token.beginswith("fid:") || + token.beginswith("fxid:") || token.beginswith("cid:") || + token.beginswith("cxid:")) { + // Save previous path if exists + if (!currentPath.empty()) { + paths.push_back(std::move(currentPath)); + } + currentPath = token.c_str(); + } else { + // Continue current path + if (!currentPath.empty()) { + currentPath += " "; + } + currentPath += token.c_str(); + } + } + } + + if (!currentPath.empty()) { + paths.push_back(std::move(currentPath)); + } - if (!rm.ParseCommand(arg)) { + if (paths.empty()) { com_rm_help(); global_retc = EINVAL; return EINVAL; } - if (rm.NeedsConfirmation() && !rm.ConfirmOperation()) { - global_retc = EINTR; - return EINTR; - } + // Execute rm for each path + int retc = 0; + for (const auto& path : paths) { + std::string cmdArg = optStr; + if (!cmdArg.empty()) { + cmdArg += " "; + } + cmdArg += path; + + RmHelper rm(gGlobalOpts); + + if (!rm.ParseCommand(cmdArg.c_str())) { + com_rm_help(); + global_retc = EINVAL; + return EINVAL; + } - global_retc = rm.Execute(true, true); + if (rm.NeedsConfirmation() && !rm.ConfirmOperation()) { + retc = EINTR; + continue; + } + + if (const int rc = rm.Execute(true, true); rc != 0) { + retc = rc; + } + } + global_retc = retc; return global_retc; } diff --git a/test/eos-instance-test b/test/eos-instance-test index c62f7d1f62..dd0a067e4b 100755 --- a/test/eos-instance-test +++ b/test/eos-instance-test @@ -1899,6 +1899,45 @@ runtest "### Stat file " unix SUCCESS "" command eos stat "$rmTestDir/regexx*p runtest "### Rm file " unix SUCCESS "" command eos rm --no-globbing "$rmTestDir/regexx*p.tt\.ern" runtest "### Stat file " unix ERROR "" command eos stat "$rmTestDir/regexx*p.tt\.ern" +# ------------------------------------------------------------------------------ +# Test multiple file removal +# ------------------------------------------------------------------------------ +RM_MULTI1="/eos/$EOS_TEST_INSTANCE/test/instancetest/multi 1.txt" +RM_MULTI2="/eos/$EOS_TEST_INSTANCE/test/instancetest/multi 2.txt" +RM_MULTI3="/eos/$EOS_TEST_INSTANCE/test/instancetest/ multi3 .txt" + +runtest "### CP " unix SUCCESS "" eos cp /etc/passwd "'${RM_MULTI1}'" +runtest "### CP " unix SUCCESS "" eos cp /etc/passwd "'${RM_MULTI2}'" +runtest "### CP " unix SUCCESS "" eos cp /etc/passwd "'${RM_MULTI3}'" +runtest "### Delay " unix SUCCESS "" delay 7 +runtest "### Rm multi " unix SUCCESS "" eos rm "'${RM_MULTI1}'" "'${RM_MULTI2}'" "'${RM_MULTI3}'" +runtest "### Delay " unix SUCCESS "" delay 7 +runtest "### Stat " unix ERROR "" eos stat "'${RM_MULTI1}'" +runtest "### Stat " unix ERROR "" eos stat "'${RM_MULTI2}'" +runtest "### Stat " unix ERROR "" eos stat "'${RM_MULTI3}'" + +# ------------------------------------------------------------------------------ +# Test multiple recursive directory removal +# ------------------------------------------------------------------------------ +RM_DIR1="/eos/$EOS_TEST_INSTANCE/test/instancetest/rmdir1" +RM_DIR2="/eos/$EOS_TEST_INSTANCE/test/instancetest/rmdir2" +RM_DIR3="/eos/$EOS_TEST_INSTANCE/test/instancetest/rmdir3" + +runtest "### Mkdir " unix SUCCESS "" eos mkdir -p ${RM_DIR1} +runtest "### Mkdir " unix SUCCESS "" eos mkdir -p ${RM_DIR2} +runtest "### Mkdir " unix SUCCESS "" eos mkdir -p ${RM_DIR3} +runtest "### Touch " unix SUCCESS "" eos touch ${RM_DIR1}/file1.txt +runtest "### Touch " unix SUCCESS "" eos touch ${RM_DIR1}/file2.txt +runtest "### Touch " unix SUCCESS "" eos touch ${RM_DIR2}/file1.txt +runtest "### Touch " unix SUCCESS "" eos touch ${RM_DIR2}/file2.txt +runtest "### Touch " unix SUCCESS "" eos touch ${RM_DIR3}/file1.txt +runtest "### Touch " unix SUCCESS "" eos touch ${RM_DIR3}/file2.txt +runtest "### Rm -r multi " unix SUCCESS "" eos rm -r ${RM_DIR1} ${RM_DIR2} ${RM_DIR3} +runtest "### Delay " unix SUCCESS "" delay 7 +runtest "### Stat " unix ERROR "" eos stat ${RM_DIR1} +runtest "### Stat " unix ERROR "" eos stat ${RM_DIR2} +runtest "### Stat " unix ERROR "" eos stat ${RM_DIR3} + # Cleanup everything runtest "### Cleanup " unix SUCCESS "" eos rm -rF $rmTestDir/ From a8504888713f87ecdbe3e1a443410998f950efef Mon Sep 17 00:00:00 2001 From: Octavian-Mihai Matei Date: Tue, 31 Mar 2026 12:22:00 +0000 Subject: [PATCH 002/113] CI: Better handling of drop deletions tests --- test/eos-instance-test | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/test/eos-instance-test b/test/eos-instance-test index dd0a067e4b..5b6ae58a5d 100755 --- a/test/eos-instance-test +++ b/test/eos-instance-test @@ -510,6 +510,17 @@ testout () { if [ $failed -gt 0 ]; then touch $FAILFILE; fi } ################################################################################ +file_is_deleted () { + local fxid=$1 + local output + output=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo "fxid:${fxid}" 2>&1) + local rc=$? + if [ $rc -ne 0 ] || echo "$output" | grep -q "pending_deletion\|tombstone\|does not exist"; then + return 0 + fi + return 1 +} +################################################################################ # Count total number of runtest calls in this script if [ "$1" != "list" ]; then echo "Counting total number of tests..." @@ -1812,11 +1823,13 @@ rm_fxid=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo ${RM_TEST_FILE} rm_fsid=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo ${RM_TEST_FILE} | jq -r .locations[].fsid | head -n 1) runtest "### Rm " unix SUCCESS "" eos rm -F ${RM_TEST_FILE} runtest "### Dropdel " unix SUCCESS "" eos fs dropdeletion "${rm_fsid}" -runtest "### Delay " unix SUCCESS "" delay 7 -runtest "### Fileinfo " unix SUCCESS "" eos fileinfo "fxid:${rm_fxid}" -runtest "### Rm detached " unix SUCCESS "" eos rm "fxid:${rm_fxid}" -runtest "### Delay " unix SUCCESS "" delay 11 -runtest "### Fileinfo " unix ERROR "" eos fileinfo "fxid:${rm_fxid}" +runtest "### Delay " unix SUCCESS "" delay 2 +if ! file_is_deleted "$rm_fxid"; then + runtest "### Fileinfo " unix SUCCESS "" eos fileinfo "fxid:${rm_fxid}" + runtest "### Rm detached " unix SUCCESS "" eos rm "fxid:${rm_fxid}" + runtest "### Delay " unix SUCCESS "" delay 2 +fi +runtest "### File Del " unix SUCCESS "" file_is_deleted "$rm_fxid" # Secondly test that rm -F on a detached file will clean up the namespace # without waiting for a confirmation from the diskservers runtest "### Upload " unix SUCCESS "" eos cp /etc/passwd ${RM_TEST_FILE} @@ -1824,10 +1837,12 @@ rm_fxid=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo ${RM_TEST_FILE} rm_fsid=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo ${RM_TEST_FILE} | jq -r .locations[].fsid | head -n 1) runtest "### Rm " unix SUCCESS "" eos rm -F ${RM_TEST_FILE} runtest "### Dropdel " unix SUCCESS "" eos fs dropdeletion "${rm_fsid}" -runtest "### Delay " unix SUCCESS "" delay 7 -runtest "### Fileinfo " unix SUCCESS "" eos fileinfo "fxid:${rm_fxid}" -runtest "### Rm force " unix SUCCESS "" eos rm -F "fxid:${rm_fxid}" -runtest "### Fileinfo " unix ERROR "" eos fileinfo "fxid:${rm_fxid}" +runtest "### Delay " unix SUCCESS "" delay 2 +if ! file_is_deleted "$rm_fxid"; then + runtest "### Fileinfo " unix SUCCESS "" eos fileinfo "fxid:${rm_fxid}" + runtest "### Rm force " unix SUCCESS "" eos rm -F "fxid:${rm_fxid}" +fi +runtest "### File Del " unix SUCCESS "" file_is_deleted "$rm_fxid" # Create a directory with 10 files in it rmTestDir="/eos/$EOS_TEST_INSTANCE/test/instancetest/rm_test" eos mkdir -p $rmTestDir From 9817c37f5ad317a4e9c00f242de6d342de37b04a Mon Sep 17 00:00:00 2001 From: Andreas Joachim Peters Date: Tue, 24 Mar 2026 13:39:33 +0100 Subject: [PATCH 003/113] CONSOLE: framework refactoring --- .gitmodules | 3 + common/Logging.hh | 7 +- console/CMakeLists.txt | 189 +- console/CommandFramework.cc | 218 + console/CommandFramework.hh | 73 + console/ConsoleArgParser.cc | 224 ++ console/ConsoleArgParser.hh | 70 + console/ConsoleCompletion.cc | 14 +- console/ConsoleMain.cc | 504 +-- console/ConsolePipe.cc | 191 - console/ConsolePipe.hh | 25 - console/README.md | 60 + console/commands/com_ls.cc | 312 -- .../commands/{ => coms/unused}/com_access.cc | 0 .../{ => coms/unused}/com_accounting.cc | 0 .../commands/{ => coms/unused}/com_archive.cc | 0 .../commands/{ => coms/unused}/com_attr.cc | 0 .../commands/{ => coms/unused}/com_backup.cc | 0 console/commands/{ => coms/unused}/com_cd.cc | 0 .../commands/{ => coms/unused}/com_chmod.cc | 0 .../commands/{ => coms/unused}/com_chown.cc | 0 .../commands/{ => coms/unused}/com_clear.cc | 0 console/commands/{ => coms/unused}/com_cp.cc | 0 .../commands/{ => coms/unused}/com_daemon.cc | 0 .../commands/{ => coms/unused}/com_debug.cc | 0 console/commands/{ => coms/unused}/com_du.cc | 4 +- .../commands/{ => coms/unused}/com_evict.cc | 0 .../commands/{ => coms/unused}/com_file.cc | 0 .../commands/{ => coms/unused}/com_fuse.cc | 0 .../commands/{ => coms/unused}/com_fusex.cc | 16 +- .../{ => coms/unused}/com_geosched.cc | 0 .../commands/{ => coms/unused}/com_health.cc | 0 .../commands/{ => coms/unused}/com_info.cc | 0 .../{ => coms/unused}/com_inspector.cc | 0 .../commands/{ => coms/unused}/com_json.cc | 0 .../commands/{ => coms/unused}/com_license.cc | 0 console/commands/{ => coms/unused}/com_ln.cc | 0 console/commands/{ => coms/unused}/com_map.cc | 0 .../commands/{ => coms/unused}/com_member.cc | 0 .../commands/{ => coms/unused}/com_mkdir.cc | 0 .../commands/{ => coms/unused}/com_motd.cc | 0 console/commands/{ => coms/unused}/com_mv.cc | 0 .../{ => coms/unused}/com_old_find.cc | 0 .../commands/{ => coms/unused}/com_print.cc | 0 .../{ => coms/unused}/com_proto_access.cc | 4 +- .../{ => coms/unused}/com_proto_acl.cc | 0 .../{ => coms/unused}/com_proto_config.cc | 0 .../{ => coms/unused}/com_proto_convert.cc | 0 .../{ => coms/unused}/com_proto_debug.cc | 0 .../{ => coms/unused}/com_proto_devices.cc | 6 +- .../{ => coms/unused}/com_proto_df.cc | 6 +- .../{ => coms/unused}/com_proto_find.cc | 0 .../{ => coms/unused}/com_proto_fs.cc | 0 .../{ => coms/unused}/com_proto_fsck.cc | 0 .../{ => coms/unused}/com_proto_group.cc | 0 .../{ => coms/unused}/com_proto_io.cc | 0 .../{ => coms/unused}/com_proto_node.cc | 0 .../{ => coms/unused}/com_proto_ns.cc | 0 .../{ => coms/unused}/com_proto_quota.cc | 0 .../{ => coms/unused}/com_proto_recycle.cc | 0 .../{ => coms/unused}/com_proto_register.cc | 6 +- .../{ => coms/unused}/com_proto_rm.cc | 0 .../{ => coms/unused}/com_proto_route.cc | 0 .../{ => coms/unused}/com_proto_sched.cc | 0 .../{ => coms/unused}/com_proto_space.cc | 0 .../{ => coms/unused}/com_proto_token.cc | 0 console/commands/{ => coms/unused}/com_pwd.cc | 0 .../commands/{ => coms/unused}/com_quit.cc | 0 .../commands/{ => coms/unused}/com_quota.cc | 4 +- .../commands/{ => coms/unused}/com_rclone.cc | 52 +- .../{ => coms/unused}/com_reconnect.cc | 0 .../commands/{ => coms/unused}/com_report.cc | 0 console/commands/{ => coms/unused}/com_rm.cc | 4 +- .../commands/{ => coms/unused}/com_rmdir.cc | 0 .../commands/{ => coms/unused}/com_role.cc | 0 .../commands/{ => coms/unused}/com_rtlog.cc | 0 .../{ => coms/unused}/com_scitoken.cc | 0 .../commands/{ => coms/unused}/com_silent.cc | 0 .../commands/{ => coms/unused}/com_squash.cc | 0 .../commands/{ => coms/unused}/com_stat.cc | 0 .../commands/{ => coms/unused}/com_status.cc | 0 .../commands/{ => coms/unused}/com_test.cc | 0 .../commands/{ => coms/unused}/com_timing.cc | 0 .../commands/{ => coms/unused}/com_touch.cc | 0 .../commands/{ => coms/unused}/com_tracker.cc | 0 .../commands/{ => coms/unused}/com_version.cc | 0 console/commands/{ => coms/unused}/com_vid.cc | 0 console/commands/{ => coms/unused}/com_who.cc | 0 .../commands/{ => coms/unused}/com_whoami.cc | 0 console/commands/helpers/AclHelper.hh | 2 +- console/commands/helpers/FsHelper.hh | 2 +- console/commands/helpers/FsckHelper.hh | 2 +- console/commands/{ => helpers}/ICmdHelper.cc | 4 +- console/commands/{ => helpers}/ICmdHelper.hh | 0 console/commands/helpers/NewfindHelper.hh | 4 +- console/commands/helpers/NodeHelper.hh | 2 +- console/commands/helpers/RecycleHelper.hh | 2 +- console/commands/helpers/TokenHelper.hh | 2 +- console/commands/native/CoreNativeCommands.cc | 177 + console/commands/native/LegacySymbols.cc | 10 + .../commands/native/access-proto-native.cc | 354 ++ .../commands/native/accounting-cmd-native.cc | 132 + console/commands/native/acl-proto-native.cc | 101 + console/commands/native/archive-cmd-native.cc | 172 + console/commands/native/attr-cmd-native.cc | 548 +++ console/commands/native/backup-cmd-native.cc | 153 + console/commands/native/cat-com-native.cc | 54 + console/commands/native/cd-cmd-native.cc | 157 + console/commands/native/chmod-cmd-native.cc | 118 + console/commands/native/chown-cmd-native.cc | 139 + console/commands/native/clear-cmd-native.cc | 92 + .../commands/native/config-proto-native.cc | 205 + .../commands/native/convert-proto-native.cc | 233 ++ console/commands/native/cp-cmd-native.cc | 1778 +++++++++ console/commands/native/daemon-native.cc | 808 ++++ console/commands/native/debug-cmd-native.cc | 184 + .../commands/native/devices-proto-native.cc | 139 + console/commands/native/df-proto-native.cc | 145 + console/commands/native/du-native.cc | 128 + console/commands/native/du-proto-native.cc | 144 + console/commands/native/evict-cmd-native.cc | 153 + console/commands/native/file-cmd-native.cc | 1135 ++++++ console/commands/native/fileinfo-alias.cc | 55 + console/commands/native/find-proto-native.cc | 154 + console/commands/native/fs-proto-native.cc | 107 + console/commands/native/fsck-proto-native.cc | 125 + console/commands/native/fuse-native.cc | 142 + console/commands/native/fusex-cmd-native.cc | 194 + .../commands/native/geosched-cmd-native.cc | 357 ++ console/commands/native/group-proto-native.cc | 168 + console/commands/native/health-native.cc | 54 + console/commands/native/info-alias.cc | 41 + console/commands/native/info-native.cc | 80 + .../commands/native/inspector-proto-native.cc | 150 + console/commands/native/io-proto-native.cc | 418 ++ console/commands/native/license-native.cc | 47 + console/commands/native/ln-cmd-native.cc | 109 + console/commands/native/ls-cmd-native.cc | 394 ++ console/commands/native/ls-compat.cc | 48 + console/commands/native/map-cmd-native.cc | 159 + console/commands/native/member-cmd-native.cc | 108 + console/commands/native/mkdir-cmd-native.cc | 109 + console/commands/native/motd-cmd-native.cc | 68 + console/commands/native/mv-alias.cc | 69 + console/commands/native/node-proto-native.cc | 100 + console/commands/native/ns-proto-native.cc | 591 +++ console/commands/native/oldfind-cmd-native.cc | 695 ++++ console/commands/native/pwd-native.cc | 43 + console/commands/native/quota-proto-native.cc | 294 ++ console/commands/native/rclone-cmd-native.cc | 1660 ++++++++ console/commands/native/reconnect-native.cc | 63 + .../commands/native/recycle-proto-native.cc | 99 + .../commands/native/register-proto-native.cc | 141 + console/commands/native/report-native.cc | 481 +++ console/commands/native/rm-proto-native.cc | 240 ++ console/commands/native/rmdir-cmd-native.cc | 103 + console/commands/native/role-native.cc | 60 + console/commands/native/route-proto-native.cc | 209 + console/commands/native/rtlog-cmd-native.cc | 131 + console/commands/native/sched-proto-native.cc | 171 + console/commands/native/scitoken-native.cc | 350 ++ console/commands/native/space-proto-native.cc | 240 ++ console/commands/native/squash-cmd-native.cc | 848 ++++ console/commands/native/stat-native.cc | 110 + console/commands/native/status-native.cc | 43 + console/commands/native/test-native.cc | 91 + console/commands/native/token-proto-native.cc | 94 + console/commands/native/touch-cmd-native.cc | 118 + .../commands/native/tracker-proto-native.cc | 102 + console/commands/native/version-cmd-native.cc | 113 + console/commands/native/vid-cmd-native.cc | 484 +++ console/commands/native/who-cmd-native.cc | 169 + console/commands/native/whoami-cmd-native.cc | 69 + console/parser | 1 + doc/diopside/my-changes.patch | 74 + doc/diopside/releases/#diopside-release.rst# | 2521 ++++++++++++ doc/diopside/releases/diopside-release.rst | 3 + fst/ScanDir.cc | 2 + mgm/#Iostat.cc# | 3503 +++++++++++++++++ mgm/inspector/FileInspector.cc | 2 +- mgm/ofs/XrdMgmOfs.hh | 10 +- mgm/quota/#Quota.cc# | 2402 +++++++++++ namespace/utils/Mode.hh | 5 +- test/eos-altxs-test | 30 +- test/eos-instance-test | 13 +- test/eos-test-utils | 28 +- unit_tests/common/StringSplitTests.cc | 15 +- 187 files changed, 27865 insertions(+), 1083 deletions(-) create mode 100644 console/CommandFramework.cc create mode 100644 console/CommandFramework.hh create mode 100644 console/ConsoleArgParser.cc create mode 100644 console/ConsoleArgParser.hh delete mode 100644 console/ConsolePipe.cc delete mode 100644 console/ConsolePipe.hh create mode 100644 console/README.md delete mode 100644 console/commands/com_ls.cc rename console/commands/{ => coms/unused}/com_access.cc (100%) rename console/commands/{ => coms/unused}/com_accounting.cc (100%) rename console/commands/{ => coms/unused}/com_archive.cc (100%) rename console/commands/{ => coms/unused}/com_attr.cc (100%) rename console/commands/{ => coms/unused}/com_backup.cc (100%) rename console/commands/{ => coms/unused}/com_cd.cc (100%) rename console/commands/{ => coms/unused}/com_chmod.cc (100%) rename console/commands/{ => coms/unused}/com_chown.cc (100%) rename console/commands/{ => coms/unused}/com_clear.cc (100%) rename console/commands/{ => coms/unused}/com_cp.cc (100%) rename console/commands/{ => coms/unused}/com_daemon.cc (100%) rename console/commands/{ => coms/unused}/com_debug.cc (100%) rename console/commands/{ => coms/unused}/com_du.cc (99%) rename console/commands/{ => coms/unused}/com_evict.cc (100%) rename console/commands/{ => coms/unused}/com_file.cc (100%) rename console/commands/{ => coms/unused}/com_fuse.cc (100%) rename console/commands/{ => coms/unused}/com_fusex.cc (98%) rename console/commands/{ => coms/unused}/com_geosched.cc (100%) rename console/commands/{ => coms/unused}/com_health.cc (100%) rename console/commands/{ => coms/unused}/com_info.cc (100%) rename console/commands/{ => coms/unused}/com_inspector.cc (100%) rename console/commands/{ => coms/unused}/com_json.cc (100%) rename console/commands/{ => coms/unused}/com_license.cc (100%) rename console/commands/{ => coms/unused}/com_ln.cc (100%) rename console/commands/{ => coms/unused}/com_map.cc (100%) rename console/commands/{ => coms/unused}/com_member.cc (100%) rename console/commands/{ => coms/unused}/com_mkdir.cc (100%) rename console/commands/{ => coms/unused}/com_motd.cc (100%) rename console/commands/{ => coms/unused}/com_mv.cc (100%) rename console/commands/{ => coms/unused}/com_old_find.cc (100%) rename console/commands/{ => coms/unused}/com_print.cc (100%) rename console/commands/{ => coms/unused}/com_proto_access.cc (99%) rename console/commands/{ => coms/unused}/com_proto_acl.cc (100%) rename console/commands/{ => coms/unused}/com_proto_config.cc (100%) rename console/commands/{ => coms/unused}/com_proto_convert.cc (100%) rename console/commands/{ => coms/unused}/com_proto_debug.cc (100%) rename console/commands/{ => coms/unused}/com_proto_devices.cc (99%) rename console/commands/{ => coms/unused}/com_proto_df.cc (99%) rename console/commands/{ => coms/unused}/com_proto_find.cc (100%) rename console/commands/{ => coms/unused}/com_proto_fs.cc (100%) rename console/commands/{ => coms/unused}/com_proto_fsck.cc (100%) rename console/commands/{ => coms/unused}/com_proto_group.cc (100%) rename console/commands/{ => coms/unused}/com_proto_io.cc (100%) rename console/commands/{ => coms/unused}/com_proto_node.cc (100%) rename console/commands/{ => coms/unused}/com_proto_ns.cc (100%) rename console/commands/{ => coms/unused}/com_proto_quota.cc (100%) rename console/commands/{ => coms/unused}/com_proto_recycle.cc (100%) rename console/commands/{ => coms/unused}/com_proto_register.cc (98%) rename console/commands/{ => coms/unused}/com_proto_rm.cc (100%) rename console/commands/{ => coms/unused}/com_proto_route.cc (100%) rename console/commands/{ => coms/unused}/com_proto_sched.cc (100%) rename console/commands/{ => coms/unused}/com_proto_space.cc (100%) rename console/commands/{ => coms/unused}/com_proto_token.cc (100%) rename console/commands/{ => coms/unused}/com_pwd.cc (100%) rename console/commands/{ => coms/unused}/com_quit.cc (100%) rename console/commands/{ => coms/unused}/com_quota.cc (99%) rename console/commands/{ => coms/unused}/com_rclone.cc (98%) rename console/commands/{ => coms/unused}/com_reconnect.cc (100%) rename console/commands/{ => coms/unused}/com_report.cc (100%) rename console/commands/{ => coms/unused}/com_rm.cc (99%) rename console/commands/{ => coms/unused}/com_rmdir.cc (100%) rename console/commands/{ => coms/unused}/com_role.cc (100%) rename console/commands/{ => coms/unused}/com_rtlog.cc (100%) rename console/commands/{ => coms/unused}/com_scitoken.cc (100%) rename console/commands/{ => coms/unused}/com_silent.cc (100%) rename console/commands/{ => coms/unused}/com_squash.cc (100%) rename console/commands/{ => coms/unused}/com_stat.cc (100%) rename console/commands/{ => coms/unused}/com_status.cc (100%) rename console/commands/{ => coms/unused}/com_test.cc (100%) rename console/commands/{ => coms/unused}/com_timing.cc (100%) rename console/commands/{ => coms/unused}/com_touch.cc (100%) rename console/commands/{ => coms/unused}/com_tracker.cc (100%) rename console/commands/{ => coms/unused}/com_version.cc (100%) rename console/commands/{ => coms/unused}/com_vid.cc (100%) rename console/commands/{ => coms/unused}/com_who.cc (100%) rename console/commands/{ => coms/unused}/com_whoami.cc (100%) rename console/commands/{ => helpers}/ICmdHelper.cc (99%) rename console/commands/{ => helpers}/ICmdHelper.hh (100%) create mode 100644 console/commands/native/CoreNativeCommands.cc create mode 100644 console/commands/native/LegacySymbols.cc create mode 100644 console/commands/native/access-proto-native.cc create mode 100644 console/commands/native/accounting-cmd-native.cc create mode 100644 console/commands/native/acl-proto-native.cc create mode 100644 console/commands/native/archive-cmd-native.cc create mode 100644 console/commands/native/attr-cmd-native.cc create mode 100644 console/commands/native/backup-cmd-native.cc create mode 100644 console/commands/native/cat-com-native.cc create mode 100644 console/commands/native/cd-cmd-native.cc create mode 100644 console/commands/native/chmod-cmd-native.cc create mode 100644 console/commands/native/chown-cmd-native.cc create mode 100644 console/commands/native/clear-cmd-native.cc create mode 100644 console/commands/native/config-proto-native.cc create mode 100644 console/commands/native/convert-proto-native.cc create mode 100644 console/commands/native/cp-cmd-native.cc create mode 100644 console/commands/native/daemon-native.cc create mode 100644 console/commands/native/debug-cmd-native.cc create mode 100644 console/commands/native/devices-proto-native.cc create mode 100644 console/commands/native/df-proto-native.cc create mode 100644 console/commands/native/du-native.cc create mode 100644 console/commands/native/du-proto-native.cc create mode 100644 console/commands/native/evict-cmd-native.cc create mode 100644 console/commands/native/file-cmd-native.cc create mode 100644 console/commands/native/fileinfo-alias.cc create mode 100644 console/commands/native/find-proto-native.cc create mode 100644 console/commands/native/fs-proto-native.cc create mode 100644 console/commands/native/fsck-proto-native.cc create mode 100644 console/commands/native/fuse-native.cc create mode 100644 console/commands/native/fusex-cmd-native.cc create mode 100644 console/commands/native/geosched-cmd-native.cc create mode 100644 console/commands/native/group-proto-native.cc create mode 100644 console/commands/native/health-native.cc create mode 100644 console/commands/native/info-alias.cc create mode 100644 console/commands/native/info-native.cc create mode 100644 console/commands/native/inspector-proto-native.cc create mode 100644 console/commands/native/io-proto-native.cc create mode 100644 console/commands/native/license-native.cc create mode 100644 console/commands/native/ln-cmd-native.cc create mode 100644 console/commands/native/ls-cmd-native.cc create mode 100644 console/commands/native/ls-compat.cc create mode 100644 console/commands/native/map-cmd-native.cc create mode 100644 console/commands/native/member-cmd-native.cc create mode 100644 console/commands/native/mkdir-cmd-native.cc create mode 100644 console/commands/native/motd-cmd-native.cc create mode 100644 console/commands/native/mv-alias.cc create mode 100644 console/commands/native/node-proto-native.cc create mode 100644 console/commands/native/ns-proto-native.cc create mode 100644 console/commands/native/oldfind-cmd-native.cc create mode 100644 console/commands/native/pwd-native.cc create mode 100644 console/commands/native/quota-proto-native.cc create mode 100644 console/commands/native/rclone-cmd-native.cc create mode 100644 console/commands/native/reconnect-native.cc create mode 100644 console/commands/native/recycle-proto-native.cc create mode 100644 console/commands/native/register-proto-native.cc create mode 100644 console/commands/native/report-native.cc create mode 100644 console/commands/native/rm-proto-native.cc create mode 100644 console/commands/native/rmdir-cmd-native.cc create mode 100644 console/commands/native/role-native.cc create mode 100644 console/commands/native/route-proto-native.cc create mode 100644 console/commands/native/rtlog-cmd-native.cc create mode 100644 console/commands/native/sched-proto-native.cc create mode 100644 console/commands/native/scitoken-native.cc create mode 100644 console/commands/native/space-proto-native.cc create mode 100644 console/commands/native/squash-cmd-native.cc create mode 100644 console/commands/native/stat-native.cc create mode 100644 console/commands/native/status-native.cc create mode 100644 console/commands/native/test-native.cc create mode 100644 console/commands/native/token-proto-native.cc create mode 100644 console/commands/native/touch-cmd-native.cc create mode 100644 console/commands/native/tracker-proto-native.cc create mode 100644 console/commands/native/version-cmd-native.cc create mode 100644 console/commands/native/vid-cmd-native.cc create mode 100644 console/commands/native/who-cmd-native.cc create mode 100644 console/commands/native/whoami-cmd-native.cc create mode 160000 console/parser create mode 100644 doc/diopside/my-changes.patch create mode 100644 doc/diopside/releases/#diopside-release.rst# create mode 100644 mgm/#Iostat.cc# create mode 100644 mgm/quota/#Quota.cc# diff --git a/.gitmodules b/.gitmodules index 4edeb35b28..bb92ce7e28 100644 --- a/.gitmodules +++ b/.gitmodules @@ -35,3 +35,6 @@ [submodule "fst/css_plugin"] path = fst/css_plugin url = https://gitlab.cern.ch/eos/css_plugin.git +[submodule "console/parser"] + path = console/parser + url = https://github.com/CLIUtils/CLI11.git diff --git a/common/Logging.hh b/common/Logging.hh index cf4892b95a..e975ecfd79 100644 --- a/common/Logging.hh +++ b/common/Logging.hh @@ -305,7 +305,9 @@ public: SetLogId(const char* newlogid) { if (newlogid && (strncmp(newlogid, logId, sizeof(logId) - 1) != 0)) { - snprintf(logId, sizeof(logId), "%s", newlogid); + // ensure bounded copy; accept truncation + strncpy(logId, newlogid, sizeof(logId) - 1); + logId[sizeof(logId) - 1] = '\0'; } } @@ -316,7 +318,8 @@ public: SetLogId(const char* newlogid, const char* td) { if (newlogid && (newlogid != logId)) { - snprintf(logId, sizeof(logId), "%s", newlogid); + strncpy(logId, newlogid, sizeof(logId) - 1); + logId[sizeof(logId) - 1] = '\0'; } if (td) { diff --git a/console/CMakeLists.txt b/console/CMakeLists.txt index 48a08e3d88..43784ea7b8 100644 --- a/console/CMakeLists.txt +++ b/console/CMakeLists.txt @@ -23,11 +23,22 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/commands/helpers/) +#------------------------------------------------------------------------------- +# CLI11 parser submodule (console/parser) +#------------------------------------------------------------------------------- +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/parser/CMakeLists.txt") + set(CLI11_BUILD_TESTS OFF CACHE BOOL "Disable CLI11 tests" FORCE) + set(CLI11_BUILD_EXAMPLES OFF CACHE BOOL "Disable CLI11 examples" FORCE) + set(CLI11_BUILD_DOCS OFF CACHE BOOL "Disable CLI11 docs" FORCE) + set(CLI11_INSTALL ON CACHE BOOL "Install CLI11 headers" FORCE) + add_subdirectory(parser) +endif() + #------------------------------------------------------------------------------- # eos executable #------------------------------------------------------------------------------- add_library(EosConsoleHelpers-Objects OBJECT - commands/ICmdHelper.cc commands/ICmdHelper.hh + commands/helpers/ICmdHelper.cc commands/helpers/ICmdHelper.hh commands/helpers/FsHelper.cc commands/helpers/FsHelper.hh commands/helpers/RecycleHelper.cc commands/helpers/RecycleHelper.hh commands/helpers/FsckHelper.cc commands/helpers/FsckHelper.hh @@ -51,97 +62,115 @@ set_target_properties(EosConsoleHelpers-Objects PROPERTIES POSITION_INDEPENDENT_CODE TRUE) add_library(EosConsoleCommands-Objects OBJECT + # Core console framework ConsoleMain.cc ConsoleMain.hh - ConsolePipe.cc ConsolePipe.hh + CommandFramework.cc CommandFramework.hh ConsoleCompletion.cc ConsoleCompletion.hh RegexUtil.cc RegexUtil.hh + + # Core/native base + commands/native/CoreNativeCommands.cc + commands/native/pwd-native.cc + commands/native/status-native.cc + + # Native aliases + commands/native/info-alias.cc + commands/native/fileinfo-alias.cc + commands/native/mv-alias.cc + commands/native/ls-compat.cc + + # Native com wrappers + commands/native/cat-com-native.cc + commands/native/cp-cmd-native.cc + commands/native/oldfind-cmd-native.cc + commands/native/rclone-cmd-native.cc + commands/native/squash-cmd-native.cc + + # Native cmd implementations (mgm.cmd=...) + commands/native/accounting-cmd-native.cc + commands/native/archive-cmd-native.cc + commands/native/attr-cmd-native.cc + commands/native/backup-cmd-native.cc + commands/native/cd-cmd-native.cc + commands/native/chmod-cmd-native.cc + commands/native/chown-cmd-native.cc + commands/native/clear-cmd-native.cc + commands/native/debug-cmd-native.cc + commands/native/evict-cmd-native.cc + commands/native/file-cmd-native.cc + commands/native/geosched-cmd-native.cc + commands/native/ln-cmd-native.cc + commands/native/ls-cmd-native.cc + commands/native/map-cmd-native.cc + commands/native/member-cmd-native.cc + commands/native/mkdir-cmd-native.cc + commands/native/motd-cmd-native.cc + commands/native/rmdir-cmd-native.cc + commands/native/rtlog-cmd-native.cc + commands/native/touch-cmd-native.cc + commands/native/version-cmd-native.cc + commands/native/vid-cmd-native.cc + commands/native/who-cmd-native.cc + commands/native/whoami-cmd-native.cc + + # Native proto implementations + commands/native/access-proto-native.cc + commands/native/acl-proto-native.cc + commands/native/config-proto-native.cc + commands/native/convert-proto-native.cc + commands/native/devices-proto-native.cc + commands/native/df-proto-native.cc + commands/native/du-proto-native.cc + commands/native/find-proto-native.cc + commands/native/fs-proto-native.cc + commands/native/fsck-proto-native.cc + commands/native/group-proto-native.cc + commands/native/inspector-proto-native.cc + commands/native/io-proto-native.cc + commands/native/node-proto-native.cc + commands/native/ns-proto-native.cc + commands/native/quota-proto-native.cc + commands/native/recycle-proto-native.cc + commands/native/register-proto-native.cc + commands/native/rm-proto-native.cc + commands/native/route-proto-native.cc + commands/native/sched-proto-native.cc + commands/native/space-proto-native.cc + commands/native/token-proto-native.cc + commands/native/tracker-proto-native.cc + + # Other native commands + commands/native/daemon-native.cc + commands/native/fuse-native.cc + commands/native/fusex-cmd-native.cc + commands/native/health-native.cc + commands/native/license-native.cc + commands/native/reconnect-native.cc + commands/native/report-native.cc + commands/native/role-native.cc + commands/native/scitoken-native.cc + commands/native/stat-native.cc + commands/native/test-native.cc + + # Legacy symbol aggregation + commands/native/LegacySymbols.cc + + # Shared helpers used by native/proto commands commands/helpers/NewfindHelper.cc commands/helpers/NewfindHelper.hh commands/helpers/AclHelper.cc commands/helpers/AclHelper.hh commands/HealthCommand.cc commands/HealthCommand.hh - commands/com_accounting.cc - commands/com_archive.cc - commands/com_attr.cc - commands/com_backup.cc - commands/com_cd.cc - commands/com_chmod.cc - commands/com_chown.cc - commands/com_clear.cc - commands/com_cp.cc - commands/com_daemon.cc - commands/com_file.cc - commands/com_old_find.cc - commands/com_proto_find.cc - commands/com_fuse.cc - commands/com_fusex.cc - commands/com_geosched.cc - commands/com_health.cc - commands/com_info.cc - commands/com_inspector.cc - commands/com_json.cc - commands/com_license.cc - commands/com_ls.cc - commands/com_ln.cc - commands/com_map.cc - commands/com_member.cc - commands/com_mkdir.cc - commands/com_motd.cc - commands/com_mv.cc - commands/com_print.cc - commands/com_pwd.cc - commands/com_quit.cc - commands/com_rclone.cc - commands/com_report.cc - commands/com_reconnect.cc - commands/com_rmdir.cc - commands/com_role.cc - commands/com_rtlog.cc - commands/com_scitoken.cc - commands/com_silent.cc - commands/com_stat.cc - commands/com_status.cc - commands/com_squash.cc - commands/com_test.cc - commands/com_timing.cc - commands/com_touch.cc - commands/com_proto_token.cc - commands/com_tracker.cc - commands/com_version.cc - commands/com_vid.cc - commands/com_whoami.cc - commands/com_who.cc - commands/com_proto_acl.cc - commands/com_proto_convert.cc - commands/com_proto_fs.cc - commands/com_proto_route.cc - commands/com_evict.cc - commands/com_proto_ns.cc - commands/com_proto_debug.cc - commands/com_du.cc - commands/com_proto_df.cc - commands/com_proto_group.cc - commands/com_proto_recycle.cc - commands/com_proto_config.cc - commands/com_proto_register.cc - commands/com_proto_rm.cc - #@todo (esindril) drop com_quota when move to 5.0.0 - commands/com_quota.cc - commands/com_proto_quota.cc - commands/com_proto_node.cc - commands/com_proto_devices.cc - commands/com_proto_space.cc - commands/com_proto_io.cc - #@todo (esindril) drop com_access when move to 5.0.0 - commands/com_access.cc - commands/com_proto_access.cc - commands/com_proto_fsck.cc - commands/com_proto_sched.cc) +) target_link_libraries(EosConsoleCommands-Objects PUBLIC EosConsoleHelpers-Objects READLINE::READLINE JSONCPP::JSONCPP SCITOKENS::SCITOKENS - XROOTD::UTILS) + XROOTD::UTILS + CLI11::CLI11) + +target_include_directories(EosConsoleCommands-Objects PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/parser/include) set_target_properties(EosConsoleCommands-Objects PROPERTIES POSITION_INDEPENDENT_CODE TRUE) diff --git a/console/CommandFramework.cc b/console/CommandFramework.cc new file mode 100644 index 0000000000..35f8ce4c42 --- /dev/null +++ b/console/CommandFramework.cc @@ -0,0 +1,218 @@ +// ---------------------------------------------------------------------- +// File: CommandFramework.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include "common/StringTokenizer.hh" +#include "common/Path.hh" +#include +#include +#include + +CommandRegistry& CommandRegistry::instance() +{ + static CommandRegistry inst; + return inst; +} + +void CommandRegistry::reg(std::unique_ptr cmd) +{ + mCommandsView.push_back(cmd.get()); + mCommands.emplace_back(std::move(cmd)); +} + +IConsoleCommand* CommandRegistry::find(const std::string& name) const +{ + // Prefer most recently registered (native overrides legacy) + for (auto it = mCommandsView.rbegin(); it != mCommandsView.rend(); ++it) { + if (name == (*it)->name()) return *it; + } + // Simple aliases + if (name == "fileinfo") { + return find("file"); + } + return nullptr; +} + +int CFuncCommandAdapter::run(const std::vector& args, CommandContext&) +{ + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + return mFunc((char*)joined.c_str()); +} + +// Default empty; concrete commands will be registered here as they are migrated +void RegisterNativeConsoleCommands() +{ + // Registration split across native modules for maintainability + // Core + extern void RegisterCoreNativeCommands(); + RegisterCoreNativeCommands(); + // Pwd + extern void RegisterPwdNativeCommand(); + RegisterPwdNativeCommand(); + // Cd + extern void RegisterCdNativeCommand(); + RegisterCdNativeCommand(); + // Ls + extern void RegisterLsNativeCommand(); + RegisterLsNativeCommand(); + // Cp + extern void RegisterCpNativeCommand(); + RegisterCpNativeCommand(); + // Version + extern void RegisterVersionNativeCommand(); + RegisterVersionNativeCommand(); + // Status + extern void RegisterStatusNativeCommand(); + RegisterStatusNativeCommand(); + // Mkdir/Rm + extern void RegisterMkdirNativeCommand(); + RegisterMkdirNativeCommand(); + extern void RegisterRmProtoNativeCommand(); + RegisterRmProtoNativeCommand(); + // Info + extern void RegisterInfoNativeCommand(); + RegisterInfoNativeCommand(); + // Stat + extern void RegisterStatNativeCommand(); + RegisterStatNativeCommand(); + // Mv + extern void RegisterMvNativeCommand(); + RegisterMvNativeCommand(); + // Ln + extern void RegisterLnNativeCommand(); + RegisterLnNativeCommand(); + // Rmdir + extern void RegisterRmdirNativeCommand(); + RegisterRmdirNativeCommand(); + // Touch + extern void RegisterTouchNativeCommand(); + RegisterTouchNativeCommand(); + // Cat + extern void RegisterCatNativeCommand(); + RegisterCatNativeCommand(); + // Who + extern void RegisterWhoNativeCommand(); + RegisterWhoNativeCommand(); + // Whoami + extern void RegisterWhoamiNativeCommand(); + RegisterWhoamiNativeCommand(); + // Proto commands + extern void RegisterAccessProtoNativeCommand(); + RegisterAccessProtoNativeCommand(); + extern void RegisterAclProtoNativeCommand(); + RegisterAclProtoNativeCommand(); + extern void RegisterConfigProtoNativeCommand(); + RegisterConfigProtoNativeCommand(); + extern void RegisterConvertProtoNativeCommand(); + RegisterConvertProtoNativeCommand(); + extern void RegisterDevicesProtoNativeCommand(); + RegisterDevicesProtoNativeCommand(); + extern void RegisterDfProtoNativeCommand(); + RegisterDfProtoNativeCommand(); + extern void RegisterFindProtoNativeCommand(); + RegisterFindProtoNativeCommand(); + extern void RegisterFsProtoNativeCommand(); + RegisterFsProtoNativeCommand(); + extern void RegisterFsckProtoNativeCommand(); + RegisterFsckProtoNativeCommand(); + extern void RegisterGroupProtoNativeCommand(); + RegisterGroupProtoNativeCommand(); + extern void RegisterIoProtoNativeCommand(); + RegisterIoProtoNativeCommand(); + extern void RegisterNodeProtoNativeCommand(); + RegisterNodeProtoNativeCommand(); + extern void RegisterNsProtoNativeCommand(); + RegisterNsProtoNativeCommand(); + extern void RegisterQuotaProtoNativeCommand(); + RegisterQuotaProtoNativeCommand(); + extern void RegisterRecycleProtoNativeCommand(); + RegisterRecycleProtoNativeCommand(); + extern void RegisterRegisterProtoNativeCommand(); + RegisterRegisterProtoNativeCommand(); + extern void RegisterRouteProtoNativeCommand(); + RegisterRouteProtoNativeCommand(); + extern void RegisterTokenProtoNativeCommand(); + RegisterTokenProtoNativeCommand(); + extern void RegisterSpaceProtoNativeCommand(); + RegisterSpaceProtoNativeCommand(); + extern void RegisterSchedProtoNativeCommand(); + RegisterSchedProtoNativeCommand(); + // file/fuse/fusex + extern void RegisterFileNativeCommand(); + RegisterFileNativeCommand(); + extern void RegisterFileInfoAliasCommand(); + RegisterFileInfoAliasCommand(); + extern void RegisterFuseNativeCommand(); + RegisterFuseNativeCommand(); + extern void RegisterFusexNativeCommand(); + RegisterFusexNativeCommand(); + // Misc + extern void RegisterBackupNativeCommand(); + RegisterBackupNativeCommand(); + extern void RegisterClearNativeCommand(); + RegisterClearNativeCommand(); + extern void RegisterDebugNativeCommand(); + RegisterDebugNativeCommand(); + extern void RegisterDuNativeCommand(); + RegisterDuNativeCommand(); + extern void RegisterEvictNativeCommand(); + RegisterEvictNativeCommand(); + extern void RegisterMotdNativeCommand(); + RegisterMotdNativeCommand(); + extern void RegisterOldfindNativeCommand(); + RegisterOldfindNativeCommand(); + extern void RegisterRcloneNativeCommand(); + RegisterRcloneNativeCommand(); + extern void RegisterSquashNativeCommand(); + RegisterSquashNativeCommand(); + extern void RegisterTestNativeCommand(); + RegisterTestNativeCommand(); + // Attr/Mode + extern void RegisterArchiveNativeCommand(); + RegisterArchiveNativeCommand(); + extern void RegisterAttrNativeCommand(); + RegisterAttrNativeCommand(); + extern void RegisterChmodNativeCommand(); + RegisterChmodNativeCommand(); + extern void RegisterChownNativeCommand(); + RegisterChownNativeCommand(); + // Admin/Device and misc extras + extern void RegisterDaemonNativeCommand(); + RegisterDaemonNativeCommand(); + extern void RegisterGeoschedNativeCommand(); + RegisterGeoschedNativeCommand(); + extern void RegisterInspectorNativeCommand(); + RegisterInspectorNativeCommand(); + extern void RegisterLicenseNativeCommand(); + RegisterLicenseNativeCommand(); + extern void RegisterMapNativeCommand(); + RegisterMapNativeCommand(); + extern void RegisterMemberNativeCommand(); + RegisterMemberNativeCommand(); + extern void RegisterAccountingNativeCommand(); + RegisterAccountingNativeCommand(); + extern void RegisterHealthNativeCommand(); + RegisterHealthNativeCommand(); + extern void RegisterReconnectNativeCommand(); + RegisterReconnectNativeCommand(); + extern void RegisterReportNativeCommand(); + RegisterReportNativeCommand(); + extern void RegisterRtlogNativeCommand(); + RegisterRtlogNativeCommand(); + extern void RegisterRoleNativeCommand(); + RegisterRoleNativeCommand(); + extern void RegisterScitokenNativeCommand(); + RegisterScitokenNativeCommand(); + extern void RegisterTrackerNativeCommand(); + RegisterTrackerNativeCommand(); + extern void RegisterVidNativeCommand(); + RegisterVidNativeCommand(); + // RemainingLegacyNativeCommands removed +} diff --git a/console/CommandFramework.hh b/console/CommandFramework.hh new file mode 100644 index 0000000000..a55e4874d2 --- /dev/null +++ b/console/CommandFramework.hh @@ -0,0 +1,73 @@ +// ---------------------------------------------------------------------- +// File: CommandFramework.hh +// Purpose: Lightweight command registry and adapters for console commands +// ---------------------------------------------------------------------- + +#pragma once + +#include +#include +#include + +#include "console/ConsoleMain.hh" + +// Context passed to commands to access existing facilities without global coupling +struct CommandContext { + std::string serverUri; + GlobalOptions* globalOpts; + bool json; + bool silent; + bool interactive; + bool timing; + std::string userRole; + std::string groupRole; + + // Thin wrappers to existing functions + XrdOucEnv* (*clientCommand)(XrdOucString& in, bool isAdmin, std::string* reply); + int (*outputResult)(XrdOucEnv* result, bool highlighting); +}; + +class IConsoleCommand { +public: + virtual ~IConsoleCommand() = default; + virtual const char* name() const = 0; + virtual const char* description() const = 0; + virtual bool requiresMgm(const std::string& args) const { return !wants_help(args.c_str()); } + virtual int run(const std::vector& args, CommandContext& ctx) = 0; + virtual void printHelp() const = 0; +}; + +class CommandRegistry { +public: + static CommandRegistry& instance(); + void reg(std::unique_ptr cmd); + IConsoleCommand* find(const std::string& name) const; + const std::vector& all() const { return mCommandsView; } + +private: + std::vector> mCommands; + std::vector mCommandsView; +}; + +// Adapter to integrate legacy C-style commands (int func(char*)) +class CFuncCommandAdapter : public IConsoleCommand { +public: + using CFunc = int(*)(char*); + CFuncCommandAdapter(const char* n, const char* d, CFunc f, bool reqMgm) + : mName(n), mDesc(d), mFunc(f), mRequiresMgm(reqMgm) {} + + const char* name() const override { return mName.c_str(); } + const char* description() const override { return mDesc.c_str(); } + bool requiresMgm(const std::string& args) const override { return mRequiresMgm && !wants_help(args.c_str()); } + int run(const std::vector& args, CommandContext&) override; + void printHelp() const override {} + +private: + std::string mName; + std::string mDesc; + CFunc mFunc; + bool mRequiresMgm; +}; + +// Register native (class-based) commands that supersede legacy ones +void RegisterNativeConsoleCommands(); diff --git a/console/ConsoleArgParser.cc b/console/ConsoleArgParser.cc new file mode 100644 index 0000000000..5eb40bd026 --- /dev/null +++ b/console/ConsoleArgParser.cc @@ -0,0 +1,224 @@ +// ---------------------------------------------------------------------- +// File: ConsoleArgParser.cc +// ---------------------------------------------------------------------- + +#include "console/ConsoleArgParser.hh" +#include + +namespace { +static bool starts_with(const std::string& s, const char* pfx) { + size_t n = 0; while (pfx[n]) ++n; if (s.size() < n) return false; return s.compare(0, n, pfx) == 0; +} +} + +ConsoleArgParser::ConsoleArgParser() = default; + +namespace { +static inline std::string dequoteToken(const std::string& in) { + if (in.size() >= 2) { + char a = in.front(); + char b = in.back(); + if ((a == '"' && b == '"') || (a == '\'' && b == '\'')) { + // avoid "" and '' edge-case returning empty meaningful token + return in.substr(1, in.size() - 2); + } + } + return in; +} +} + +ConsoleArgParser& ConsoleArgParser::setProgramName(const std::string& nm) { + mProgramName = nm; return *this; +} + +ConsoleArgParser& ConsoleArgParser::setDescription(const std::string& desc) { + mDescription = desc; return *this; +} + +ConsoleArgParser& ConsoleArgParser::allowCombinedShortOptions(bool allow) { + mAllowCombinedShorts = allow; return *this; +} + +ConsoleArgParser& ConsoleArgParser::allowAttachedValue(bool allow) { + mAllowAttachedValue = allow; return *this; +} + +ConsoleArgParser& ConsoleArgParser::acceptBareAssignments(bool accept) { + mAcceptBareAssignments = accept; return *this; +} + +ConsoleArgParser& ConsoleArgParser::collectUnknownTokens(bool collect) { + mCollectUnknownTokens = collect; return *this; +} + +ConsoleArgParser& ConsoleArgParser::addOption(const OptionSpec& spec) { + InternalSpec is; is.spec = spec; + size_t idx = mSpecs.size(); + mSpecs.push_back(is); + if (!spec.longName.empty()) mLongToIndex.emplace(spec.longName, idx); + if (spec.shortName) mShortToIndex.emplace(spec.shortName, idx); + return *this; +} + +const ConsoleArgParser::InternalSpec* ConsoleArgParser::findByLong(const std::string& nm) const { + auto it = mLongToIndex.find(nm); + if (it == mLongToIndex.end()) return nullptr; + return &mSpecs[it->second]; +} + +const ConsoleArgParser::InternalSpec* ConsoleArgParser::findByShort(char c) const { + auto it = mShortToIndex.find(c); + if (it == mShortToIndex.end()) return nullptr; + return &mSpecs[it->second]; +} + +bool ConsoleArgParser::ParseResult::has(const std::string& name) const { + return optionToValues.find(name) != optionToValues.end(); +} + +bool ConsoleArgParser::ParseResult::flag(const std::string& name) const { + auto it = optionToValues.find(name); + if (it == optionToValues.end()) return false; + return it->second.empty(); +} + +std::string ConsoleArgParser::ParseResult::value(const std::string& name, const std::string& fallback) const { + auto it = optionToValues.find(name); + if (it == optionToValues.end() || it->second.empty()) return fallback; + return it->second.back(); +} + +std::vector ConsoleArgParser::ParseResult::values(const std::string& name) const { + auto it = optionToValues.find(name); + if (it == optionToValues.end()) return {}; + return it->second; +} + +ConsoleArgParser::ParseResult ConsoleArgParser::parse(const std::vector& args) const { + ParseResult r; + bool onlyPositionals = false; + + auto add_value = [&](const InternalSpec* spec, const std::string& v) { + if (!spec) return; + auto& vec = r.optionToValues[spec->spec.longName.empty() ? std::string(1, spec->spec.shortName) : spec->spec.longName]; + vec.push_back(v); + }; + + auto set_flag = [&](const InternalSpec* spec){ add_value(spec, std::string()); }; + + for (size_t i = 0; i < args.size(); ++i) { + std::string tok = dequoteToken(args[i]); + if (onlyPositionals) { r.positionals.push_back(tok); continue; } + + if (tok == "--") { onlyPositionals = true; continue; } + + // Long option: --opt or --opt=value + if (starts_with(tok, "--")) { + std::string nameval = dequoteToken(tok.substr(2)); + std::string name, val; + size_t eq = nameval.find('='); + if (eq == std::string::npos) name = nameval; else { name = nameval.substr(0, eq); val = dequoteToken(nameval.substr(eq+1)); } + auto* s = findByLong(name); + if (!s) { if (mCollectUnknownTokens) r.unknownTokens.push_back(tok); else r.errors.push_back("Unknown option: " + tok); continue; } + if (s->spec.requiresValue) { + if (!val.empty()) { add_value(s, val); } + else { + if (i+1 < args.size()) { add_value(s, dequoteToken(args[++i])); } + else { r.errors.push_back("Missing value for option --" + name); } + } + } else { + set_flag(s); + } + continue; + } + + // Short option(s): -a -abc -oValue -o Value + if (tok.size() >= 2 && tok[0] == '-' && tok[1] != '-') { + // Combined or single + if (mAllowCombinedShorts && tok.size() > 2) { + // iterate every char after '-' + for (size_t k = 1; k < tok.size(); ++k) { + char c = tok[k]; + auto* s = findByShort(c); + if (!s) { if (mCollectUnknownTokens) r.unknownTokens.push_back(std::string("-") + c); else r.errors.push_back(std::string("Unknown option: -") + c); continue; } + if (s->spec.requiresValue) { + std::string val; + if (mAllowAttachedValue && k+1 < tok.size()) { + val = tok.substr(k+1); + add_value(s, val); + break; // rest belongs to value + } else if (i+1 < args.size()) { + add_value(s, dequoteToken(args[++i])); + } else { + r.errors.push_back(std::string("Missing value for option -") + c); + } + } else { + set_flag(s); + } + } + } else { + // Single short option like -o or -oValue + char c = tok.size() > 1 ? tok[1] : '\0'; + auto* s = findByShort(c); + if (!s) { if (mCollectUnknownTokens) r.unknownTokens.push_back(tok); else r.errors.push_back("Unknown option: " + tok); continue; } + if (s->spec.requiresValue) { + std::string val; + if (mAllowAttachedValue && tok.size() > 2) val = tok.substr(2); + else if (i+1 < args.size()) val = dequoteToken(args[++i]); + else r.errors.push_back(std::string("Missing value for option -") + c); + if (!val.empty()) add_value(s, val); + } else { + set_flag(s); + } + } + continue; + } + + // Bare assignment key=value (legacy style) + if (mAcceptBareAssignments) { + size_t eq = tok.find('='); + if (eq != std::string::npos && eq > 0) { + std::string key = dequoteToken(tok.substr(0, eq)); + std::string val = dequoteToken(tok.substr(eq+1)); + // if there is a spec matching the long name, map it; otherwise keep as positional assignment + if (auto* s = findByLong(key)) { + add_value(s, val); + } else { + r.positionals.push_back(tok); + } + continue; + } + } + + // Positional + r.positionals.push_back(tok); + } + + // Apply defaults + for (const auto& ins : mSpecs) { + const auto& spec = ins.spec; + if (!spec.defaultValue.empty() && r.optionToValues.find(spec.longName) == r.optionToValues.end()) { + r.optionToValues[spec.longName].push_back(spec.defaultValue); + } + } + + return r; +} + +std::string ConsoleArgParser::help() const { + std::ostringstream oss; + if (!mProgramName.empty()) oss << "Usage: " << mProgramName << " [options] [--] [args...]\n"; + if (!mDescription.empty()) oss << mDescription << "\n\n"; + oss << "Options:\n"; + for (const auto& ins : mSpecs) { + const auto& s = ins.spec; + oss << " "; + if (s.shortName) oss << '-' << s.shortName << (s.longName.empty() ? "" : ", "); + if (!s.longName.empty()) oss << "--" << s.longName; + if (s.requiresValue) oss << ' ' << (s.valueName.empty() ? "" : s.valueName); + if (!s.description.empty()) oss << "\t" << s.description; + if (!s.defaultValue.empty()) oss << " (default: " << s.defaultValue << ")"; + oss << "\n"; + } + return oss.str(); +} diff --git a/console/ConsoleArgParser.hh b/console/ConsoleArgParser.hh new file mode 100644 index 0000000000..d4dcd2d586 --- /dev/null +++ b/console/ConsoleArgParser.hh @@ -0,0 +1,70 @@ +// ---------------------------------------------------------------------- +// File: ConsoleArgParser.hh +// ---------------------------------------------------------------------- + +#pragma once + +#include +#include +#include +#include +#include + +class ConsoleArgParser { +public: + struct OptionSpec { + std::string longName; + char shortName = '\0'; + bool requiresValue = false; + bool allowMultiple = false; + std::string valueName; + std::string description; + std::string defaultValue; + }; + + struct ParseResult { + std::unordered_map> optionToValues; + std::vector positionals; + std::vector unknownTokens; + std::vector errors; + + bool has(const std::string& name) const; + bool flag(const std::string& name) const; // present without a required value + std::string value(const std::string& name, const std::string& fallback = "") const; + std::vector values(const std::string& name) const; + }; + + ConsoleArgParser(); + + ConsoleArgParser& setProgramName(const std::string& nm); + ConsoleArgParser& setDescription(const std::string& desc); + ConsoleArgParser& allowCombinedShortOptions(bool allow); + ConsoleArgParser& allowAttachedValue(bool allow); + ConsoleArgParser& acceptBareAssignments(bool accept); // e.g. key=value without dashes + ConsoleArgParser& collectUnknownTokens(bool collect); + + ConsoleArgParser& addOption(const OptionSpec& spec); + + ParseResult parse(const std::vector& args) const; + + std::string help() const; + +private: + struct InternalSpec { + OptionSpec spec; + }; + + const InternalSpec* findByLong(const std::string& nm) const; + const InternalSpec* findByShort(char c) const; + + std::string mProgramName; + std::string mDescription; + bool mAllowCombinedShorts = true; + bool mAllowAttachedValue = true; + bool mAcceptBareAssignments = true; + bool mCollectUnknownTokens = true; + + std::vector mSpecs; + std::unordered_map mLongToIndex; + std::unordered_map mShortToIndex; +}; diff --git a/console/ConsoleCompletion.cc b/console/ConsoleCompletion.cc index 82447be7f4..2162927418 100644 --- a/console/ConsoleCompletion.cc +++ b/console/ConsoleCompletion.cc @@ -22,6 +22,7 @@ #include "ConsoleMain.hh" #include "ConsoleCompletion.hh" +#include "CommandFramework.hh" #include #include #include @@ -226,15 +227,12 @@ char* eos_command_generator(const char* text, int state) int len = strlen(text); completions.clear(); index = 0; - char* name; - int i = 0; - while ((name = commands[i].name)) { - if (strncmp(name, text, len) == 0) { - completions.push_back(name); - } - - ++i; + // Prefer registry over static commands array + auto& all = CommandRegistry::instance().all(); + for (auto* c : all) { + const char* name = c->name(); + if (strncmp(name, text, len) == 0) completions.push_back(name); } } else { ++index; diff --git a/console/ConsoleMain.cc b/console/ConsoleMain.cc index 23d439537a..17e8884816 100644 --- a/console/ConsoleMain.cc +++ b/console/ConsoleMain.cc @@ -22,15 +22,14 @@ ************************************************************************/ #include "ConsoleMain.hh" -#include "ConsolePipe.hh" #include "ConsoleCompletion.hh" +#include "CommandFramework.hh" #include "console/RegexUtil.hh" #include #include #include "License" #include "common/FileId.hh" #include "common/Path.hh" -#include "common/IoPipe.hh" #include "common/SymKeys.hh" #include "common/StringTokenizer.hh" #include "common/StringConversion.hh" @@ -53,173 +52,6 @@ #define ENONET 64 #endif -//------------------------------------------------------------------------------ -// Implemented commands -//------------------------------------------------------------------------------ -extern int com_protoaccess(char*); -extern int com_acl(char*); -extern int com_archive(char*); -extern int com_attr(char*); -extern int com_backup(char*); -extern int com_cat(char*); -extern int com_cd(char*); -extern int com_chmod(char*); -extern int com_chown(char*); -extern int com_clear(char*); -extern int com_protoconfig(char*); -extern int com_convert(char*); -extern int com_cp(char*); -extern int com_protodebug(char*); -extern int com_du(char*); -extern int com_protodf(char*); -extern int com_file(char*); -extern int com_fileinfo(char*); -extern int com_old_find(char*); -extern int com_proto_find(char*); -extern int com_protofs(char*); -extern int com_proto_fsck(char*); -extern int com_fuse(char*); -extern int com_fusex(char*); -extern int com_geosched(char*); -extern int com_protogroup(char*); -extern int com_health(char*); -extern int com_help(char*); -extern int com_info(char*); -extern int com_inspector(char*); -extern int com_protoio(char*); -extern int com_json(char*); -extern int com_license(char*); -extern int com_ln(char*); -extern int com_ls(char*); -extern int com_map(char*); -extern int com_member(char*); -extern int com_mkdir(char*); -extern int com_motd(char*); -extern int com_mv(char*); -extern int com_protonode(char*); -extern int com_ns(char*); -extern int com_pwd(char*); -extern int com_quit(char*); -extern int com_protoquota(char*); -extern int com_report(char*); -extern int com_reconnect(char*); -extern int com_protorecycle(char*); -extern int com_rm(char*); -extern int com_route(char*); -extern int com_protorm(char*); -extern int com_protoregister(char*); -extern int com_rclone(char*); -extern int com_rmdir(char*); -extern int com_role(char*); -extern int com_rtlog(char*); -extern int com_scitoken(char*); -extern int com_status(char*); -extern int com_silent(char*); -extern int com_proto_devices(char*); -extern int com_proto_space(char*); -extern int com_evict(char*); -extern int com_stat(char*); -extern int com_squash(char*); -extern int com_test(char*); -extern int com_timing(char*); -extern int com_tracker(char*); -extern int com_touch(char*); -extern int com_proto_token(char*); -extern int com_prot_token(char*); -extern int com_version(char*); -extern int com_vid(char*); -extern int com_whoami(char*); -extern int com_who(char*); -extern int com_accounting(char*); -extern int com_quota(char*); -extern int com_daemon(char*); -extern int com_proto_sched(char*); - -//------------------------------------------------------------------------------ -// Command mapping array -//------------------------------------------------------------------------------ -COMMAND commands[] = { - { (char*) "access", com_protoaccess, (char*) "Access Interface"}, - { (char*) "accounting", com_accounting, (char*) "Accounting Interface"}, - { (char*) "acl", com_acl, (char*) "Acl Interface"}, - { (char*) "archive", com_archive, (char*) "Archive Interface"}, - { (char*) "attr", com_attr, (char*) "Attribute Interface"}, - { (char*) "backup", com_backup, (char*) "Backup Interface"}, - { (char*) "clear", com_clear, (char*) "Clear the terminal"}, - { (char*) "cat", com_cat, (char*) "Cat a file"}, - { (char*) "cd", com_cd, (char*) "Change directory"}, - { (char*) "chmod", com_chmod, (char*) "Mode Interface"}, - { (char*) "chown", com_chown, (char*) "Chown Interface"}, - { (char*) "config", com_protoconfig, (char*) "Configuration System"}, - { (char*) "convert", com_convert, (char*) "Convert Interface"}, - { (char*) "cp", com_cp, (char*) "Cp command"}, - { (char*) "daemon", com_daemon, (char*) "Handle service daemon"}, - { (char*) "debug", com_protodebug, (char*) "Set debug level"}, - { (char*) "devices", com_proto_devices, (char*) "Get Device Information"}, - { (char*) "du", com_du, (char*) "Get du output"}, - { (char*) "df", com_protodf, (char*) "Get df output"}, - { (char*) "exit", com_quit, (char*) "Exit from EOS console"}, - { (char*) "file", com_file, (char*) "File Handling"}, - { (char*) "fileinfo", com_fileinfo, (char*) "File Information"}, - { (char*) "oldfind", com_old_find, (char*) "Find files/directories (old implementation)"}, - { (char*) "find", com_proto_find, (char*) "Find files/directories (new implementation)"}, - { (char*) "newfind", com_proto_find, (char*) "Find files/directories (new implementation)"}, - { (char*) "fs", com_protofs, (char*) "File System configuration"}, - { (char*) "fsck", com_proto_fsck, (char*) "File System Consistency Checking"}, - { (char*) "fuse", com_fuse, (char*) "Fuse Mounting"}, - { (char*) "fusex", com_fusex, (char*) "Fuse(x) Administration"}, - { (char*) "geosched", com_geosched, (char*) "Geoscheduler Interface"}, - { (char*) "group", com_protogroup, (char*) "Group configuration"}, - { (char*) "health", com_health, (char*) "Health information about system"}, - { (char*) "help", com_help, (char*) "Display this text"}, - { (char*) "info", com_info, (char*) "Retrieve file or directory information"}, - { (char*) "inspector", com_inspector, (char*) "Interact with File Inspector"}, - { (char*) "io", com_protoio, (char*) "IO Interface"}, - { (char*) "json", com_json, (char*) "Toggle JSON output flag for stdout"}, - { (char*) "license", com_license, (char*) "Display Software License"}, - { (char*) "ls", com_ls, (char*) "List a directory"}, - { (char*) "ln", com_ln, (char*) "Create a symbolic link"}, - { (char*) "map", com_map, (char*) "Path mapping interface"}, - { (char*) "member", com_member, (char*) "Check Egroup membership"}, - { (char*) "mkdir", com_mkdir, (char*) "Create a directory"}, - { (char*) "motd", com_motd, (char*) "Message of the day"}, - { (char*) "mv", com_mv, (char*) "Rename file or directory"}, - { (char*) "node", com_protonode, (char*) "Node configuration"}, - { (char*) "ns", com_ns, (char*) "Namespace Interface"}, - { (char*) "pwd", com_pwd, (char*) "Print working directory"}, - { (char*) "quit", com_quit, (char*) "Exit from EOS console"}, - { (char*) "quota", com_protoquota, (char*) "Quota System configuration"}, - { (char*) "rclone", com_rclone, (char*) "RClone like command"}, - { (char*) "report", com_report, (char*) "Analyze report log files on the local machine"}, - { (char*) "reconnect", com_reconnect, (char*) "Forces a re-authentication of the shell"}, - { (char*) "recycle", com_protorecycle, (char*) "Recycle Bin Functionality"}, - { (char*) "register", com_protoregister, (char*) "Register a file"}, - { (char*) "rmdir", com_rmdir, (char*) "Remove a directory"}, - { (char*) "rm", com_protorm, (char*) "Remove a file"}, - { (char*) "role", com_role, (char*) "Set the client role"}, - { (char*) "route", com_route, (char*) "Routing interface"}, - { (char*) "rtlog", com_rtlog, (char*) "Get realtime log output from mgm & fst servers"}, - { (char*) "sched", com_proto_sched, (char*) "Configure the various scheduler options"}, - { (char*) "silent", com_silent, (char*) "Toggle silent flag for stdout"}, - { (char*) "status", com_status, (char*) "Display status information on an MGM"}, - { (char*) "space", com_proto_space, (char*) "Space configuration"}, - { (char*) "evict", com_evict, (char*) "Evict disk replicas of a file if it has tape replicas"}, - { (char*) "stat", com_stat, (char*) "Run 'stat' on a file or directory"}, - { (char*) "squash", com_squash, (char*) "Run 'squashfs' utility function"}, - { (char*) "test", com_test, (char*) "Run performance test"}, - { (char*) "timing", com_timing, (char*) "Toggle timing flag for execution time measurement"}, - { (char*) "touch", com_touch, (char*) "Touch a file"}, - { (char*) "token", com_proto_token, (char*) "Token interface"}, - { (char*) "scitoken", com_scitoken, (char*) "SciToken interface"}, - { (char*) "tracker", com_tracker, (char*) "Interact with File Tracker"}, - { (char*) "version", com_version, (char*) "Verbose client/server version"}, - { (char*) "vid", com_vid, (char*) "Virtual ID System Configuration"}, - { (char*) "whoami", com_whoami, (char*) "Determine how we are mapped on server side"}, - { (char*) "who", com_who, (char*) "Statistics about connected users"}, - { (char*) "?", com_help, (char*) "Synonym for 'help'"}, - { (char*) ".q", com_quit, (char*) "Exit from EOS console"}, - { (char*) 0, (int (*)(char*))0, (char*) 0} -}; //------------------------------------------------------------------------------ // Global variables @@ -243,14 +75,8 @@ bool interactive = true; bool hasterminal = true; bool silent = false; bool timing = false; -bool pipemode = false; -bool runpipe = false; -bool ispipe = false; bool json = false; GlobalOptions gGlobalOpts; - -eos::common::IoPipe iopipe; -int retcfd = 0; //! When non-zero, this global means the user is done using this program. */ int done; @@ -259,6 +85,56 @@ int done; XrdOucEnv* CommandEnv = 0; static sigjmp_buf sigjump_buf; +// Registry initialization helper +static bool registryInitialized = false; + +static void EnsureRegistryInitialized() +{ + if (!registryInitialized) { + RegisterNativeConsoleCommands(); + registryInitialized = true; + } +} + +// Convenience runner using the registered native commands +static int RunRegisteredCommand(const std::string& cmdName, + const std::vector& argsVec) +{ + EnsureRegistryInitialized(); + IConsoleCommand* icmd = CommandRegistry::instance().find(cmdName); + if (!icmd) { + fprintf(stderr, "%s: No such command for EOS Console.\n", cmdName.c_str()); + return -1; + } + + std::string rest; + for (size_t i = 0; i < argsVec.size(); ++i) { + if (i) rest.push_back(' '); + rest += argsVec[i]; + } + + if (icmd->requiresMgm(rest) && + !CheckMgmOnline(serveruri.c_str())) { + std::cerr << "error: MGM " << serveruri.c_str() + << " not online/reachable" << std::endl; + exit(ENONET); + } + + CommandContext ctx; + ctx.serverUri = serveruri.c_str(); + ctx.globalOpts = &gGlobalOpts; + ctx.json = json; + ctx.silent = silent; + ctx.interactive = interactive; + ctx.timing = timing; + ctx.userRole = user_role.c_str(); + ctx.groupRole = group_role.c_str(); + ctx.clientCommand = &client_command; + ctx.outputResult = &output_result; + + return icmd->run(argsVec, ctx); +} + //------------------------------------------------------------------------------ // Exit handler //------------------------------------------------------------------------------ @@ -268,11 +144,6 @@ exit_handler(int a) fprintf(stdout, "\n"); fprintf(stderr, "\n"); write_history(historyfile.c_str()); - - if (ispipe) { - iopipe.UnLockProducer(); - } - exit(-1); } @@ -294,6 +165,12 @@ abspath(const char* in) static XrdOucString inpath; inpath = in; + if (inpath.beginswith("fxid:") || inpath.beginswith("fid:") || + inpath.beginswith("cxid:") || inpath.beginswith("cid:") || + inpath.beginswith("pxid:") || inpath.beginswith("pid:")) { + return inpath.c_str(); + } + if (inpath.beginswith("/")) { return inpath.c_str(); } @@ -373,54 +250,6 @@ wants_help(const char* args_line, bool no_h) return false; } -//------------------------------------------------------------------------------ -// Switches stdin, stdout, stderr to pipe mode where we are a persistent -// communication daemon for a the eospipe command forwarding commands. -//------------------------------------------------------------------------------ -bool -startpipe() -{ - XrdOucString pipedir = ""; - XrdOucString stdinname = ""; - XrdOucString stdoutname = ""; - XrdOucString stderrname = ""; - XrdOucString retcname = ""; - ispipe = true; - close(STDIN_FILENO); - close(STDOUT_FILENO); - close(STDERR_FILENO); - - if (!iopipe.Init()) { - fprintf(stderr, "error: cannot set IoPipe\n"); - return false; - } - - XrdSysLogger* logger = new XrdSysLogger(); - XrdSysError eDest(logger); - int stdinfd = iopipe.AttachStdin(eDest); - int stdoutfd = iopipe.AttachStdout(eDest); - int stderrfd = iopipe.AttachStderr(eDest); - retcfd = iopipe.AttachRetc(eDest); - - if ((stdinfd < 0) || - (stdoutfd < 0) || - (stderrfd < 0) || - (retcfd < 0)) { - fprintf(stderr, "error: cannot attach to pipes\n"); - return false; - } - - if (!iopipe.LockProducer()) { - return false; - } - - stdin = fdopen(stdinfd, "r"); - stdout = fdopen(stdoutfd, "w"); - stderr = fdopen(stderrfd, "w"); - return true; -} - - /* **************************************************************** */ /* */ /* EOSConsole Commands */ @@ -718,7 +547,7 @@ read_pwdfile() eos::common::StringConversion::LoadFileIntoString(pwdfile.c_str(), lpwd); if (lpwd.length()) { - com_cd((char*) lpwd.c_str()); + RunRegisteredCommand("cd", {lpwd}); } } @@ -745,17 +574,15 @@ usage() fprintf(stderr, "`eos' is the command line interface (CLI) of the EOS storage system.\n"); fprintf(stderr, - "Usage: eos [-r|--role ] [-s] [-a|--app ] [-b|--batch] [-v|--version] [-p|--pipe] [-j|--json] [] [ {}|.eosh]\n"); + "Usage: eos [-r|--role ] [-s] [-a|--app ] [-b|--batch] [-v|--version] [-j|--json] [] [ {}|.eosh]\n"); fprintf(stderr, " -r, --role : select user role and group role \n"); fprintf(stderr, " -a, --app : set the application name for the CLI\n"); fprintf(stderr, - " -b, --batch : run in batch mode without colour and syntax highlighting and without pipe\n"); + " -b, --batch : run in batch mode without colour and syntax highlighting\n"); fprintf(stderr, " -j, --json : switch to json output format\n"); - fprintf(stderr, - " -p, --pipe : run stdin,stdout,stderr on local pipes and go to background\n"); fprintf(stderr, " -h, --help : print help text\n"); fprintf(stderr, @@ -779,8 +606,6 @@ usage() " EOS_NEWGRP : requests for each command the group ID of the current shell\n"); fprintf(stderr, " EOS_PWD_FILE : sets the file where the last working directory is stored- by default '$HOME/.eos_pwd\n\n"); - fprintf(stderr, - " EOS_ENABLE_PIPEMODE : allows the EOS shell to split into a session and pipe executable to avoid useless re-authentication\n"); fprintf(stderr, "Return Value: \n"); fprintf(stderr, " The return code of the last executed command is returned. 0 is returned in case of success otherwise (!=0).\n\n"); @@ -848,12 +673,6 @@ Run(int argc, char* argv[]) int argindex = 1; int retc = system("test -t 0 && test -t 1"); - if (getenv("EOS_ENABLE_PIPEMODE")) { - runpipe = true; - } else { - runpipe = false; - } - if (getenv("EOS_CONSOLE_DEBUG")) { global_debug = true; gGlobalOpts.mDebug = true; @@ -876,13 +695,11 @@ Run(int argc, char* argv[]) if ((in1 != "--help") && (in1 != "--version") && (in1 != "--batch") && - (in1 != "--pipe") && (in1 != "--role") && (in1 != "--json") && (in1 != "--app") && (in1 != "-h") && (in1 != "-b") && - (in1 != "-p") && (in1 != "-v") && (in1 != "-s") && (in1 != "-j") && @@ -899,19 +716,18 @@ Run(int argc, char* argv[]) } if ((in1 == "-s")) { - return com_status(0); + return RunRegisteredCommand("status", {}); } if ((in1 == "--version") || (in1 == "-v")) { - fprintf(stderr, "EOS %s (2020)\n\n", VERSION); - fprintf(stderr, "Developed by the CERN IT storage group\n"); + fprintf(stderr, "EOS %s (2026)\n\n", VERSION); + fprintf(stderr, "Developed by the CERN IT Storage Group\n"); exit(0); } if ((in1 == "--batch") || (in1 == "-b")) { interactive = false; global_highlighting = false; - runpipe = false; argindex++; in1 = argv[argindex]; } @@ -920,7 +736,6 @@ Run(int argc, char* argv[]) interactive = false; global_highlighting = false; json = true; - runpipe = false; argindex++; in1 = argv[argindex]; gGlobalOpts.mJsonFormat = true; @@ -929,19 +744,6 @@ Run(int argc, char* argv[]) if ((in1 == "fuse")) { interactive = false; global_highlighting = false; - runpipe = false; - } - - if ((in1 == "--pipe") || (in1 == "-p")) { - pipemode = true; - argindex++; - in1 = argv[argindex]; - - if (!startpipe()) { - fprintf(stderr, "error: unable to start the pipe - maybe there is " - "already a process with 'eos -p' running?\n"); - exit(-1); - } } if ((in1 == "--role") || (in1 == "-r")) { @@ -984,10 +786,9 @@ Run(int argc, char* argv[]) in1 = argv[argindex]; } - if ((in1 == "cp")) { + if ((in1 == "cp") || (in1 == "rclone")) { interactive = false; global_highlighting = false; - runpipe = false; } if ((in1 == "fuse")) { @@ -1077,47 +878,8 @@ Run(int argc, char* argv[]) cmdline.erase(cmdline.length() - 1, 1); } - // Here we can use the 'eospipe' mechanism if allowed - if (runpipe) { - cmdline += "\n"; - // put the eos daemon into batch mode - interactive = false; - global_highlighting = false; - iopipe.Init(); // need to initialize for CheckProducer - - if (!iopipe.CheckProducer()) { - // We need to run a pipe daemon, so we fork here and let the fork - // run the code like 'eos -p' - if (!fork()) { - for (int i = 1; i < argc; i++) { - for (size_t j = 0; j < strlen(argv[i]); j++) { - argv[i][j] = '*'; - } - } - - // detach from the session id - pid_t sid; - - if ((sid = setsid()) < 0) { - fprintf(stderr, "ERROR: failed to create new session (setsid())\n"); - exit(-1); - } - - startpipe(); - pipemode = true; - // enters down the readline loop with modified stdin,stdout,stderr - } else { - // now we just deal with the pipes from the client end - exit(pipe_command(cmdline.c_str())); - } - } else { - // now we just deal with the pipes from the client end - exit(pipe_command(cmdline.c_str())); - } - } else { - execute_line((char*) cmdline.c_str()); - exit(global_retc); - } + execute_line((char*) cmdline.c_str()); + exit(global_retc); } } } @@ -1177,12 +939,8 @@ Run(int argc, char* argv[]) } char prompt[4096]; - if (pipemode) { - prompt[0] = 0; - } else { - sprintf(prompt, "%sEOS Console%s [%s%s%s] |> ", textbold.c_str(), - textunbold.c_str(), textred.c_str(), serveruri.c_str(), textnormal.c_str()); - } + sprintf(prompt, "%sEOS Console%s [%s%s%s] |> ", textbold.c_str(), + textunbold.c_str(), textred.c_str(), serveruri.c_str(), textnormal.c_str()); // Bind our completer rl_readline_name = (char*) "EOS Console"; @@ -1219,18 +977,9 @@ Run(int argc, char* argv[]) for (; done == 0;) { char prompt[4096]; - if (pipemode) { - prompt[0] = 0; - } else { - sprintf(prompt, "%sEOS Console%s [%s%s%s] |%s> ", textbold.c_str(), - textunbold.c_str(), textred.c_str(), serveruri.c_str(), textnormal.c_str(), - gPwd.c_str()); - } - - if (pipemode) { - signal(SIGALRM, exit_handler); - alarm(60); - } + sprintf(prompt, "%sEOS Console%s [%s%s%s] |%s> ", textbold.c_str(), + textunbold.c_str(), textred.c_str(), serveruri.c_str(), textnormal.c_str(), + gPwd.c_str()); signal(SIGINT, jump_handler); @@ -1242,10 +991,6 @@ Run(int argc, char* argv[]) line = readline(prompt); signal(SIGINT, exit_handler); - if (pipemode) { - alarm(0); - } - if (!line) { fprintf(stdout, "\n"); break; @@ -1262,28 +1007,10 @@ Run(int argc, char* argv[]) alarm(3600); execute_line(s); alarm(0); - char newline = '\n'; - int n = 0; std::cout << std::flush; std::cerr << std::flush; fflush(stdout); fflush(stderr); - - if (pipemode) { - n = write(retcfd, &global_retc, sizeof(global_retc)); - n = write(retcfd, &newline, sizeof(newline)); - - if (n != 1) { - fprintf(stderr, "error: unable to write retc to retc-socket\n"); - exit(-1); - } - - // we send the stop sequence to the pipe thread listeners - fprintf(stdout, "#__STOP__#\n"); - fprintf(stderr, "#__STOP__#\n"); - fflush(stdout); - fflush(stderr); - } } free(line); @@ -1321,45 +1048,81 @@ execute_line(char* line) return (-1); } - COMMAND* command = find_command(tokens.begin()->c_str()); + // Initialize registry on first use + EnsureRegistryInitialized(); - if (!command) { - fprintf(stderr, "%s: No such command for EOS Console.\n", - tokens.begin()->c_str()); + std::string cmdName = *tokens.begin(); + IConsoleCommand* icmd = CommandRegistry::instance().find(cmdName); + if (!icmd) { + fprintf(stderr, "%s: No such command for EOS Console.\n", cmdName.c_str()); global_retc = -1; return (-1); } - // Extract arguments string from full command line + // Extract arguments vector from full command line line_without_comment = line_without_comment.substr(tokens.begin()->size()); eos::common::trim(line_without_comment); - std::string args = line_without_comment; + std::string rest = line_without_comment; + // Quote-aware tokenization: preserve spaces within quoted strings, drop quotes + std::vector argsVec; + { + bool inD = false, inS = false; + std::string cur; + for (size_t i = 0; i < rest.size(); ++i) { + char c = rest[i]; + if (c == '\\' && i + 1 < rest.size()) { + char next = rest[i + 1]; + if (next == '"' || next == '\'') { + // Preserve escaped quotes as literals and avoid toggling state. + cur.push_back(c); + cur.push_back(next); + ++i; + continue; + } + } + if (c == '"' && !inS) { + inD = !inD; + continue; + } + if (c == '\'' && !inD) { + inS = !inS; + continue; + } + if (c == ' ' && !inD && !inS) { + if (!cur.empty()) { + argsVec.push_back(cur); + cur.clear(); + } + continue; + } + cur.push_back(c); + } + if (!cur.empty()) { + argsVec.push_back(cur); + } + } // Check MGM availability - if (RequiresMgm(command->name, args) && + if (icmd->requiresMgm(rest) && !CheckMgmOnline(serveruri.c_str())) { std::cerr << "error: MGM " << serveruri.c_str() << " not online/reachable" << std::endl; exit(ENONET); } - return ((*(command->func))((char*)args.c_str())); -} - -//------------------------------------------------------------------------------ -// Look up NAME as the name of a command, and return a pointer to that command. -// Return a 0 pointer if NAME isn't a command name. -//------------------------------------------------------------------------------ -COMMAND* -find_command(const char* name) -{ - for (int i = 0; commands[i].name; ++i) { - if (strcmp(name, commands[i].name) == 0) { - return (&commands[i]); - } - } + CommandContext ctx; + ctx.serverUri = serveruri.c_str(); + ctx.globalOpts = &gGlobalOpts; + ctx.json = json; + ctx.silent = silent; + ctx.interactive = interactive; + ctx.timing = timing; + ctx.userRole = user_role.c_str(); + ctx.groupRole = group_role.c_str(); + ctx.clientCommand = &client_command; + ctx.outputResult = &output_result; - return ((COMMAND*) 0); + return icmd->run(argsVec, ctx); } //------------------------------------------------------------------------------ @@ -1631,7 +1394,6 @@ std::string DefaultRoute() //------------------------------------------------------------------------------ int filesystems::Load(bool verbose) { - std::string cmd = "ls -m -s"; struct stat buf; std::string cachefile = "/tmp/.eos.filesystems."; XrdOucString serverflat = serveruri; @@ -1656,7 +1418,7 @@ int filesystems::Load(bool verbose) rstdout = eos::common::StringConversion::LoadFileIntoString(cachefile.c_str(), out); } else { - retc = com_protofs((char*)cmd.c_str()); + retc = RunRegisteredCommand("fs", {"-m", "-s"}); std::string in = rstdout.c_str(); eos::common::StringConversion::SaveStringIntoFile(cachefiletmp.c_str(), in); ::rename(cachefiletmp.c_str(), cachefile.c_str()); @@ -1715,7 +1477,9 @@ int files::Find(const char* path, bool verbose) cmd += path; bool old_silent = silent; silent = true; - int retc = com_proto_find((char*)cmd.c_str()); + int retc = RunRegisteredCommand("find", + {"-f", "--nrep", "--fid", "--fs", "--checksum", + "--size", path}); silent = old_silent; if (!retc) { diff --git a/console/ConsolePipe.cc b/console/ConsolePipe.cc deleted file mode 100644 index 35de654af0..0000000000 --- a/console/ConsolePipe.cc +++ /dev/null @@ -1,191 +0,0 @@ -// ---------------------------------------------------------------------- -// File: ConsolePipe.cc -// Author: Andreas-Joachim Peters - CERN -// ---------------------------------------------------------------------- - -/************************************************************************ - * EOS - the CERN Disk Storage System * - * Copyright (C) 2011 CERN/Switzerland * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation, either version 3 of the License, or * - * (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program. If not, see .* - ************************************************************************/ - -/*----------------------------------------------------------------------------*/ -#include "ConsolePipe.hh" -#include "common/Path.hh" -#include "common/IoPipe.hh" -/*----------------------------------------------------------------------------*/ -#include -#include -#include -#include -#include -/*----------------------------------------------------------------------------*/ -#include -#include -#include -#include -/*----------------------------------------------------------------------------*/ - -extern eos::common::IoPipe iopipe; - -static void* -StaticThreadReaderStdout (void* arg) -{ - int fd = ((unsigned long long) arg) % 65536; - - XrdOucString sline = ""; - do - { - char c; - int nread = read(fd, &c, 1); - if (nread == 1) - { - sline += c; - if (c == '\n') - { - if (sline.find("#__STOP__#") != STR_NPOS) - { - sline.replace("#__STOP__#\n", ""); - fprintf(stdout, "%s", sline.c_str()); - return 0; - } - fprintf(stdout, "%s", sline.c_str()); - sline = ""; - } - } - else - { - fprintf(stderr, "socket read failed on fd %d\n", fd); - } - } - while (1); - return 0; -} - -static void* -StaticThreadReaderStderr (void* arg) -{ - int fd = ((unsigned long long) arg) % 65536; - - XrdOucString sline = ""; - - do - { - char c; - int nread = read(fd, &c, 1); - if (nread == 1) - { - sline += c; - if (c == '\n') - { - if (sline.find("#__STOP__#\n") != STR_NPOS) - { - sline.replace("#__STOP__#\n", ""); - fprintf(stderr, "%s", sline.c_str()); - return 0; - } - fprintf(stderr, "%s", sline.c_str()); - sline = ""; - } - } - else - { - fprintf(stderr, "socket read failed on fd %d\n", fd); - } - } - while (1); - - return 0; -} - -void -pipe_exit_handler (int a) -{ - fprintf(stdout, "\n"); - fprintf(stderr, "\n"); - iopipe.KillProducer(); - iopipe.UnLockConsumer(); - exit(-1); -} - -int -pipe_command (const char* cmd) -{ - - if (!cmd) - { - return -1; - } - - XrdSysLogger* logger = new XrdSysLogger(); - XrdSysError eDest(logger); - - if (!iopipe.Init()) - { - fprintf(stderr, "error: cannot set IoPipe\n"); - return -1; - } - - iopipe.LockConsumer(); - - int stdinfd = iopipe.AttachStdin(eDest); - int stdoutfd = iopipe.AttachStdout(eDest); - int stderrfd = iopipe.AttachStderr(eDest); - int retcfd = iopipe.AttachRetc(eDest); - - - if ((stdinfd < 0) || - (stdoutfd < 0) || - (stderrfd < 0) || - (retcfd < 0)) - { - fprintf(stderr, "error: cannot attache to pipes\n"); - return -1; - } - - pthread_t thread1; - pthread_t thread2; - - XrdSysThread::Run(&thread1, StaticThreadReaderStdout, (void*) (unsigned long long)stdoutfd, XRDSYSTHREAD_HOLD, "Stdout Thread"); - XrdSysThread::Run(&thread2, StaticThreadReaderStderr, (void*) (unsigned long long)stderrfd, XRDSYSTHREAD_HOLD, "Stderr Thread"); - - signal(SIGINT, pipe_exit_handler); - signal(SIGPIPE, SIG_IGN); - - int n = write(stdinfd, cmd, strlen(cmd)); - if (n != (int) strlen(cmd)) - { - fprintf(stderr, "error: communication error to the connector - write failed errno=%d\n", errno); - } - - XrdSysThread::Join(thread1, NULL); - XrdSysThread::Join(thread2, NULL); - - // read the response code - char a[2]; - n = read(retcfd, a, 2); - - if (n != 2) - { - fprintf(stderr, "error: socket read failed on fd %d\n", retcfd); - pipe_exit_handler(-1); - return -1; // we don't get here anyway - } - else - { - iopipe.UnLockConsumer(); - return (a[0]); - } -} diff --git a/console/ConsolePipe.hh b/console/ConsolePipe.hh deleted file mode 100644 index 52e87b9b71..0000000000 --- a/console/ConsolePipe.hh +++ /dev/null @@ -1,25 +0,0 @@ -// ---------------------------------------------------------------------- -// File: ConsolePipe.hh -// Author: Andreas-Joachim Peters - CERN -// ---------------------------------------------------------------------- - -/************************************************************************ - * EOS - the CERN Disk Storage System * - * Copyright (C) 2011 CERN/Switzerland * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation, either version 3 of the License, or * - * (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program. If not, see .* - ************************************************************************/ - - -extern int pipe_command (const char* cmd); diff --git a/console/README.md b/console/README.md new file mode 100644 index 0000000000..3049e420f6 --- /dev/null +++ b/console/README.md @@ -0,0 +1,60 @@ +## EOS Console Command Framework (Native) + +All console commands are dispatched through `CommandRegistry`/`IConsoleCommand`. The legacy `commands[]` table is gone; registration happens only via per-file `Register*` functions. + +### Directory layout (console/) +- `CommandFramework.*` : Registry, interfaces, and core wiring +- `ConsoleMain.*` : REPL and dispatch; calls `RegisterNativeConsoleCommands()` +- `commands/native/` : Native command implementations + - `*-proto-native.cc` : Proto-based commands + - `*-cmd-native.cc` : Direct MGM (`mgm.cmd=...`) commands + - `*-com-native.cc` : Wrappers around legacy `com_*` + - `*-alias.cc` : Aliases forwarding to other commands +- `commands/helpers/` : Shared helpers (e.g., `ICmdHelper`, FsHelper, TokenHelper) +- `commands/coms/` : Legacy `com_*.cc` sources + - `commands/coms/unused/` : Legacy sources no longer linked/needed +- `commands/native/LegacySymbols.cc` : Minimal legacy symbols still required by some native wrappers +- `console/CMakeLists.txt` : Build list; add new native files here + +### Core interfaces +- `IConsoleCommand`: `name()`, `description()`, `requiresMgm(args)`, `run(args, CommandContext&)`, `printHelp()`. +- `CommandContext`: current CLI state plus callbacks `clientCommand(...)` and `outputResult(...)`. Prefer these over direct legacy globals. +- `CommandRegistry`: `reg(...)`, `find(name)`, `all()`. Newest registration wins, so native overrides any legacy adapters. + +### Naming conventions for native files +- `*-proto-native.cc` : builds/uses protobuf helpers (`SpaceProto_*`, etc.). +- `*-cmd-native.cc` : builds MGM requests directly (`mgm.cmd=...`). +- `*-com-native.cc` : wraps or delegates to legacy `com_*` implementations. +- `*-alias.cc` : forwards to another command (e.g., `info-alias` → `file info`). + +### Registering commands +- Each file exposes `RegisterXxx...()` and calls `CommandRegistry::instance().reg(...)`. +- `CommandFramework.cc` invokes all `Register*` once at startup (`RegisterNativeConsoleCommands()`). +- Add new files to `console/CMakeLists.txt` under `EosConsoleCommands-Objects`. + +### Calling another console command +Use the registry to compose/forward behavior: +```c++ +IConsoleCommand* other = CommandRegistry::instance().find("file"); +if (!other) { fprintf(stderr, "error: 'file' command not available\n"); return EINVAL; } +std::vector forwarded = {"info", path}; +int rc = other->run(forwarded, ctx); // reuse the same CommandContext +``` + +### Creating an alias +- Create a small `*-alias.cc`. +- In `run(...)`, find the target command, prepend the subcommand, and call `run` on the target with the same `CommandContext`. +- Implement `printHelp()` with a `usage:` line (emit to `stderr`) noting it forwards to the target. + +### Help / usage +- `printHelp()` should write to `stderr` and begin with `usage: ...`. +- On invalid args or `--help`, call `printHelp()`, set `global_retc = EINVAL`, and return. + +### MGM requirement +- Return `false` from `requiresMgm(...)` for local-only commands (e.g., `clear`, `whoami`); otherwise typically return `!wants_help(args.c_str())` so help does not ping MGM. + +### Migrating legacy commands +- If a native command still calls `com_*`, keep the file named `*-com-native.cc`. +- If it builds `mgm.cmd=...`, use `*-cmd-native.cc`. +- If it uses protobuf helpers, use `*-proto-native.cc`. +- Move logic into `run(...)` using `CommandContext` callbacks; drop legacy dependencies once behavior matches. diff --git a/console/commands/com_ls.cc b/console/commands/com_ls.cc deleted file mode 100644 index 4fd1357c1e..0000000000 --- a/console/commands/com_ls.cc +++ /dev/null @@ -1,312 +0,0 @@ -// ---------------------------------------------------------------------- -// File: com_ls.cc -// Author: Andreas-Joachim Peters - CERN -// ---------------------------------------------------------------------- - -/************************************************************************ - * EOS - the CERN Disk Storage System * - * Copyright (C) 2011 CERN/Switzerland * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation, either version 3 of the License, or * - * (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU General Public License for more details. * - * * - * You should have received a copy of the GNU General Public License * - * along with this program. If not, see .* - ************************************************************************/ - -#include "console/ConsoleMain.hh" -#include "common/StringTokenizer.hh" -#include "common/StringConversion.hh" -#include -#include -#include "namespace/utils/Mode.hh" -#include "stdio.h" - -/* List a directory */ -int -com_ls(char* arg1) -{ - // split subcommands - eos::common::StringTokenizer subtokenizer(arg1); - subtokenizer.GetLine(); - XrdOucString param = ""; - XrdOucString option = ""; - XrdOucString path = ""; - XrdOucString in = "mgm.cmd=ls"; - - do { - param = subtokenizer.GetToken(false); - - if (!param.length()) { - break; - } - - if (param == "--help") { - goto com_ls_usage; - } - - if (param == "-h") { - goto com_ls_usage; - } - - if (param.beginswith("-")) { - option += param; - - if (param == "-c") { - option += "-l"; - } - - if ((option.find("&")) != STR_NPOS) { - goto com_ls_usage; - } - } else { - if (path.length()) { - // this allows for spaces in path names - path += " "; - path += param; - } else { - path = param; - } - } - } while (1); - - if (!path.length()) { - path = gPwd; - } - - // remove escaped blanks - while (path.replace("\\ ", " ")) { - } - - if ((path.beginswith("as3:"))) { - // extract evt. the hostname - // the hostname is part of the URL like in ROOT - XrdOucString hostport; - XrdOucString protocol; - XrdOucString sPath; - const char* v = 0; - - if (!(v = eos::common::StringConversion::ParseUrl(path.c_str(), protocol, - hostport))) { - fprintf(stderr, "error: illegal url <%s>\n", path.c_str()); - global_retc = EINVAL; - return (0); - } - - sPath = v; - - if (hostport.length()) { - setenv("S3_HOSTNAME", hostport.c_str(), 1); - } - - XrdOucString envString = path; - int qpos = 0; - - if ((qpos = envString.find("?")) != STR_NPOS) { - envString.erase(0, qpos + 1); - XrdOucEnv env(envString.c_str()); - - // extract opaque S3 tags if present - if (env.Get("s3.key")) { - setenv("S3_SECRET_ACCESS_KEY", env.Get("s3.key"), 1); - } - - if (env.Get("s3.id")) { - setenv("S3_ACCESS_KEY_ID", env.Get("s3.id"), 1); - } - - path.erase(path.find("?")); - sPath.erase(sPath.find("?")); - } - - // Apply the ROOT compatability environment variables - const char* cstr = getenv("S3_ACCESS_KEY"); - - if (cstr) { - setenv("S3_SECRET_ACCESS_KEY", cstr, 1); - } - - cstr = getenv("S3_ACESSS_ID"); - - if (cstr) { - setenv("S3_ACCESS_KEY_ID", cstr, 1); - } - - // check that the environment is set - if (!getenv("S3_ACCESS_KEY_ID") || - !getenv("S3_HOSTNAME") || - !getenv("S3_SECRET_ACCESS_KEY")) { - fprintf(stderr, - "error: you have to set the S3 environment variables S3_ACCESS_KEY_ID | S3_ACCESS_ID, S3_HOSTNAME (or use a URI), S3_SECRET_ACCESS_KEY | S3_ACCESS_KEY\n"); - exit(-1); - } - - XrdOucString s3env; - s3env = "env S3_ACCESS_KEY_ID="; - s3env += getenv("S3_ACCESS_KEY_ID"); - s3env += " S3_HOSTNAME="; - s3env += getenv("S3_HOSTNAME"); - s3env += " S3_SECRET_ACCESS_KEY="; - s3env += getenv("S3_SECRET_ACCESS_KEY"); - XrdOucString s3arg = sPath.c_str(); - // do some bash magic ... sigh - XrdOucString listcmd = "bash -c \""; - listcmd += s3env; - listcmd += " s3 list "; - listcmd += s3arg; - listcmd += " "; - listcmd += "\""; - global_retc = system(listcmd.c_str()); - return (0); - } - - if ((path.beginswith("file:")) || (path.beginswith("root:"))) { - // list a local or XRootD path - bool XRootD = path.beginswith("root:"); - XrdOucString protocol; - XrdOucString hostport; - XrdOucString sPath; - const char* v = 0; - - if (!(v = eos::common::StringConversion::ParseUrl(path.c_str(), protocol, - hostport))) { - global_retc = EINVAL; - return (0); - } - - sPath = v; - std::string Path = v; - - if (sPath == "" && (protocol == "file")) { - sPath = getenv("PWD"); - Path = getenv("PWD"); - - if (!sPath.endswith("/")) { - sPath += "/"; - Path += "/"; - } - } - - XrdOucString url = ""; - eos::common::StringConversion::CreateUrl(protocol.c_str(), hostport.c_str(), - Path.c_str(), url); - DIR* dir = (XRootD) ? XrdPosixXrootd::Opendir(url.c_str()) : opendir( - url.c_str()); - - if (dir) { - struct dirent* entry; - - while ((entry = (XRootD) ? XrdPosixXrootd::Readdir(dir) : readdir(dir))) { - struct stat buf; - XrdOucString curl = ""; - XrdOucString cpath = Path.c_str(); - cpath += entry->d_name; - eos::common::StringConversion::CreateUrl(protocol.c_str(), hostport.c_str(), - cpath.c_str(), curl); - - if ((option.find("a")) == STR_NPOS) { - if (entry->d_name[0] == '.') { - // skip hidden files - continue; - } - } - - if (!((XRootD) ? XrdPosixXrootd::Stat(curl.c_str(), &buf) : stat(curl.c_str(), - &buf))) { - if ((option.find("l")) == STR_NPOS) { - // no details - fprintf(stdout, "%s\n", entry->d_name); - } else { - char t_creat[14]; - char modestr[11]; - eos::modeToBuffer(buf.st_mode, modestr); - XrdOucString suid = ""; - suid += (int) buf.st_uid; - XrdOucString sgid = ""; - sgid += (int) buf.st_gid; - XrdOucString sizestring = ""; - struct tm* t_tm; - struct tm t_tm_local; - t_tm = localtime_r(&buf.st_ctime, &t_tm_local); - strftime(t_creat, 13, "%b %d %H:%M", t_tm); - XrdOucString dirmarker = ""; - - if ((option.find("F")) != STR_NPOS) { - dirmarker = "/"; - } - - if (modestr[0] != 'd') { - dirmarker = ""; - } - - fprintf(stdout, "%s %3d %-8.8s %-8.8s %12s %s %s%s\n", modestr, - (int) buf.st_nlink, - suid.c_str(), sgid.c_str(), - eos::common::StringConversion::GetSizeString(sizestring, - (unsigned long long) buf.st_size), t_creat, entry->d_name, dirmarker.c_str()); - } - } - } - - (XRootD) ? XrdPosixXrootd::Closedir(dir) : closedir(dir); - } - - global_retc = 0; - return (0); - } - - path = abspath(path.c_str()); - - if (strlen(path.c_str()) >= FILENAME_MAX) { - fprintf(stderr, "error: path length longer than %i bytes", FILENAME_MAX); - global_retc = EINVAL; - return (0); - } - - path = eos::common::StringConversion::curl_escaped(path.c_str()).c_str(); - in += "&mgm.path="; - in += path; - in += "&eos.encodepath=1"; - in += "&mgm.option="; - in += option; - global_retc = output_result(client_command(in)); - return (0); -com_ls_usage: - fprintf(stdout, - "usage: ls [-laniyFN] [--no-globbing] \n"); - fprintf(stdout, "list directory \n\n"); - fprintf(stdout, - " -l : show long listing\n"); - fprintf(stdout, - " -y : show long listing with backend(tape) status\n"); - fprintf(stdout, - " -lh: show long listing with readable sizes\n"); - fprintf(stdout, - " -a : show hidden files\n"); - fprintf(stdout, - " -i : add inode information\n"); - fprintf(stdout, - " -c : add checksum value\n"); - fprintf(stdout, - " -n : show numerical user/group ids\n"); - fprintf(stdout, - " -F : append indicator '/' to directories \n"); - fprintf(stdout, - " -s : checks only if the directory exists without listing\n"); - fprintf(stdout, - " --no-globbing|-N : disables path globbing feature (e.g: list a file containing '[]' characters)\n"); - fprintf(stdout, " path=file:... : list on a local file system\n\n"); - fprintf(stdout, - "path=root:... : list on a plain XRootD server (does not work on native XRootD clusters\n"); - fprintf(stdout, - "path=... : all other paths are considered to be EOS paths!\n"); - global_retc = EINVAL; - return (0); -} diff --git a/console/commands/com_access.cc b/console/commands/coms/unused/com_access.cc similarity index 100% rename from console/commands/com_access.cc rename to console/commands/coms/unused/com_access.cc diff --git a/console/commands/com_accounting.cc b/console/commands/coms/unused/com_accounting.cc similarity index 100% rename from console/commands/com_accounting.cc rename to console/commands/coms/unused/com_accounting.cc diff --git a/console/commands/com_archive.cc b/console/commands/coms/unused/com_archive.cc similarity index 100% rename from console/commands/com_archive.cc rename to console/commands/coms/unused/com_archive.cc diff --git a/console/commands/com_attr.cc b/console/commands/coms/unused/com_attr.cc similarity index 100% rename from console/commands/com_attr.cc rename to console/commands/coms/unused/com_attr.cc diff --git a/console/commands/com_backup.cc b/console/commands/coms/unused/com_backup.cc similarity index 100% rename from console/commands/com_backup.cc rename to console/commands/coms/unused/com_backup.cc diff --git a/console/commands/com_cd.cc b/console/commands/coms/unused/com_cd.cc similarity index 100% rename from console/commands/com_cd.cc rename to console/commands/coms/unused/com_cd.cc diff --git a/console/commands/com_chmod.cc b/console/commands/coms/unused/com_chmod.cc similarity index 100% rename from console/commands/com_chmod.cc rename to console/commands/coms/unused/com_chmod.cc diff --git a/console/commands/com_chown.cc b/console/commands/coms/unused/com_chown.cc similarity index 100% rename from console/commands/com_chown.cc rename to console/commands/coms/unused/com_chown.cc diff --git a/console/commands/com_clear.cc b/console/commands/coms/unused/com_clear.cc similarity index 100% rename from console/commands/com_clear.cc rename to console/commands/coms/unused/com_clear.cc diff --git a/console/commands/com_cp.cc b/console/commands/coms/unused/com_cp.cc similarity index 100% rename from console/commands/com_cp.cc rename to console/commands/coms/unused/com_cp.cc diff --git a/console/commands/com_daemon.cc b/console/commands/coms/unused/com_daemon.cc similarity index 100% rename from console/commands/com_daemon.cc rename to console/commands/coms/unused/com_daemon.cc diff --git a/console/commands/com_debug.cc b/console/commands/coms/unused/com_debug.cc similarity index 100% rename from console/commands/com_debug.cc rename to console/commands/coms/unused/com_debug.cc diff --git a/console/commands/com_du.cc b/console/commands/coms/unused/com_du.cc similarity index 99% rename from console/commands/com_du.cc rename to console/commands/coms/unused/com_du.cc index ada801d254..fdd3d43578 100644 --- a/console/commands/com_du.cc +++ b/console/commands/coms/unused/com_du.cc @@ -48,7 +48,7 @@ com_du(char* arg) bool printreadable = false; bool printsummary = false; bool printsi = false; - + std::string path; do { @@ -93,7 +93,7 @@ com_du(char* arg) } cmd += " "; cmd += path; - + return com_proto_find((char*)cmd.c_str()); } diff --git a/console/commands/com_evict.cc b/console/commands/coms/unused/com_evict.cc similarity index 100% rename from console/commands/com_evict.cc rename to console/commands/coms/unused/com_evict.cc diff --git a/console/commands/com_file.cc b/console/commands/coms/unused/com_file.cc similarity index 100% rename from console/commands/com_file.cc rename to console/commands/coms/unused/com_file.cc diff --git a/console/commands/com_fuse.cc b/console/commands/coms/unused/com_fuse.cc similarity index 100% rename from console/commands/com_fuse.cc rename to console/commands/coms/unused/com_fuse.cc diff --git a/console/commands/com_fusex.cc b/console/commands/coms/unused/com_fusex.cc similarity index 98% rename from console/commands/com_fusex.cc rename to console/commands/coms/unused/com_fusex.cc index fa56a19b9e..7e6cf29ab5 100644 --- a/console/commands/com_fusex.cc +++ b/console/commands/coms/unused/com_fusex.cc @@ -119,7 +119,7 @@ com_fusex(char* arg1) int i_interval = interval.length() ? atoi(interval.c_str()) : 0; int q_interval = quota_interval.length() ? atoi(quota_interval.c_str()) : 0; - + if ((i_interval < 0) || (i_interval > 60)) { goto com_fusex_usage; @@ -218,14 +218,14 @@ com_fusex(char* arg1) fprintf(stdout, " - if reason contains the keyword 'log2big' the client will effectily not be evicted, but will truncate his logfile to 0\n"); - fprintf(stdout, + fprintf(stdout, " - if reason contains the keyword 'setlog' and 'debug','notice', 'error', 'crit', 'info', 'warning' the log level of the targeted mount is changed accordingly .e.g evict \"setlog error\"\n"); - - fprintf(stdout, + + fprintf(stdout, " - if reason contains the keyword 'stacktrace' the client will send a self-stacktrace with the next heartbeat message and it will be stored in /var/log/eos/mgm/eosxd-stacktraces.log e.g. evict stacktrace\n"); - fprintf(stdout, + fprintf(stdout, " - if reason contains the keyword 'sendlog' the client will send max. the last 512 lines of each log level and the log will be stored in /var/log/eos/mgm/eosxd-logtraces.log e.g. evict sendlog\n"); - fprintf(stdout, + fprintf(stdout, " - if reason contains the keyword 'resetbuffer' the client will reset the read-ahead and write-buffers in flight and possibly unlock a locked mount point"); fprintf(stdout, "\n"); @@ -261,7 +261,7 @@ com_fusex(char* arg1) " fusex conf [] [quota-check-in-seconds] [max broadcast audience] [broadcast audience match]\n"); fprintf(stdout, " : show heartbeat and quota interval\n"); - + fprintf(stdout, " : [ optional change heartbeat interval from [1-15] seconds ]\n"); fprintf(stdout, @@ -273,7 +273,7 @@ com_fusex(char* arg1) " fusex conf 10 : define heartbeat interval as 10 seconds\n"); fprintf(stdout, " fusex conf 10 30 : define heartbeat as 10 seconds and quota interval as 30 seconds\n"); - fprintf(stdout, + fprintf(stdout, " fusex conf 0 0 256 @b[67] : suppress broadcasts when more than 256 clients are conected and the target matches @b[67]\n"); return (0); } diff --git a/console/commands/com_geosched.cc b/console/commands/coms/unused/com_geosched.cc similarity index 100% rename from console/commands/com_geosched.cc rename to console/commands/coms/unused/com_geosched.cc diff --git a/console/commands/com_health.cc b/console/commands/coms/unused/com_health.cc similarity index 100% rename from console/commands/com_health.cc rename to console/commands/coms/unused/com_health.cc diff --git a/console/commands/com_info.cc b/console/commands/coms/unused/com_info.cc similarity index 100% rename from console/commands/com_info.cc rename to console/commands/coms/unused/com_info.cc diff --git a/console/commands/com_inspector.cc b/console/commands/coms/unused/com_inspector.cc similarity index 100% rename from console/commands/com_inspector.cc rename to console/commands/coms/unused/com_inspector.cc diff --git a/console/commands/com_json.cc b/console/commands/coms/unused/com_json.cc similarity index 100% rename from console/commands/com_json.cc rename to console/commands/coms/unused/com_json.cc diff --git a/console/commands/com_license.cc b/console/commands/coms/unused/com_license.cc similarity index 100% rename from console/commands/com_license.cc rename to console/commands/coms/unused/com_license.cc diff --git a/console/commands/com_ln.cc b/console/commands/coms/unused/com_ln.cc similarity index 100% rename from console/commands/com_ln.cc rename to console/commands/coms/unused/com_ln.cc diff --git a/console/commands/com_map.cc b/console/commands/coms/unused/com_map.cc similarity index 100% rename from console/commands/com_map.cc rename to console/commands/coms/unused/com_map.cc diff --git a/console/commands/com_member.cc b/console/commands/coms/unused/com_member.cc similarity index 100% rename from console/commands/com_member.cc rename to console/commands/coms/unused/com_member.cc diff --git a/console/commands/com_mkdir.cc b/console/commands/coms/unused/com_mkdir.cc similarity index 100% rename from console/commands/com_mkdir.cc rename to console/commands/coms/unused/com_mkdir.cc diff --git a/console/commands/com_motd.cc b/console/commands/coms/unused/com_motd.cc similarity index 100% rename from console/commands/com_motd.cc rename to console/commands/coms/unused/com_motd.cc diff --git a/console/commands/com_mv.cc b/console/commands/coms/unused/com_mv.cc similarity index 100% rename from console/commands/com_mv.cc rename to console/commands/coms/unused/com_mv.cc diff --git a/console/commands/com_old_find.cc b/console/commands/coms/unused/com_old_find.cc similarity index 100% rename from console/commands/com_old_find.cc rename to console/commands/coms/unused/com_old_find.cc diff --git a/console/commands/com_print.cc b/console/commands/coms/unused/com_print.cc similarity index 100% rename from console/commands/com_print.cc rename to console/commands/coms/unused/com_print.cc diff --git a/console/commands/com_proto_access.cc b/console/commands/coms/unused/com_proto_access.cc similarity index 99% rename from console/commands/com_proto_access.cc rename to console/commands/coms/unused/com_proto_access.cc index 2fe3fbaf05..e660185f08 100644 --- a/console/commands/com_proto_access.cc +++ b/console/commands/coms/unused/com_proto_access.cc @@ -312,11 +312,11 @@ bool AccessHelper::ParseCommand(const char* arg) } else { return false; } - + if (!tokenizer.NextToken(token)) { return false; } - + if (token == "stall") { stall->set_type(eos::console::AccessProto_StallHostsProto::STALL); } else if (token == "nostall") { diff --git a/console/commands/com_proto_acl.cc b/console/commands/coms/unused/com_proto_acl.cc similarity index 100% rename from console/commands/com_proto_acl.cc rename to console/commands/coms/unused/com_proto_acl.cc diff --git a/console/commands/com_proto_config.cc b/console/commands/coms/unused/com_proto_config.cc similarity index 100% rename from console/commands/com_proto_config.cc rename to console/commands/coms/unused/com_proto_config.cc diff --git a/console/commands/com_proto_convert.cc b/console/commands/coms/unused/com_proto_convert.cc similarity index 100% rename from console/commands/com_proto_convert.cc rename to console/commands/coms/unused/com_proto_convert.cc diff --git a/console/commands/com_proto_debug.cc b/console/commands/coms/unused/com_proto_debug.cc similarity index 100% rename from console/commands/com_proto_debug.cc rename to console/commands/coms/unused/com_proto_debug.cc diff --git a/console/commands/com_proto_devices.cc b/console/commands/coms/unused/com_proto_devices.cc similarity index 99% rename from console/commands/com_proto_devices.cc rename to console/commands/coms/unused/com_proto_devices.cc index 812a8308e9..be9d1959c2 100644 --- a/console/commands/com_proto_devices.cc +++ b/console/commands/coms/unused/com_proto_devices.cc @@ -68,14 +68,14 @@ DevicesHelper::ParseCommand(const char* arg) eos::console::DevicesProto* devices = mReq.mutable_devices(); eos::common::StringTokenizer tokenizer(arg); tokenizer.GetLine(); - + if (!tokenizer.NextToken(token)) { return false; } - + eos::console::DevicesProto_LsProto* ls = devices->mutable_ls(); ls->set_outformat(eos::console::DevicesProto_LsProto::NONE); - + if (token == "ls") { do { tokenizer.NextToken(token); diff --git a/console/commands/com_proto_df.cc b/console/commands/coms/unused/com_proto_df.cc similarity index 99% rename from console/commands/com_proto_df.cc rename to console/commands/coms/unused/com_proto_df.cc index b7e98643ce..d5e90608ca 100644 --- a/console/commands/com_proto_df.cc +++ b/console/commands/coms/unused/com_proto_df.cc @@ -68,10 +68,10 @@ bool DfHelper::ParseCommand(const char* arg) eos::common::StringTokenizer tokenizer(arg); tokenizer.GetLine(); std::string token; - + dfproto->set_si(true); dfproto->set_readable(true); - + if (!tokenizer.NextToken(token)) { return true; } @@ -105,7 +105,7 @@ bool DfHelper::ParseCommand(const char* arg) } path = token; } - + if (tokenizer.NextToken(token)) { return false; } diff --git a/console/commands/com_proto_find.cc b/console/commands/coms/unused/com_proto_find.cc similarity index 100% rename from console/commands/com_proto_find.cc rename to console/commands/coms/unused/com_proto_find.cc diff --git a/console/commands/com_proto_fs.cc b/console/commands/coms/unused/com_proto_fs.cc similarity index 100% rename from console/commands/com_proto_fs.cc rename to console/commands/coms/unused/com_proto_fs.cc diff --git a/console/commands/com_proto_fsck.cc b/console/commands/coms/unused/com_proto_fsck.cc similarity index 100% rename from console/commands/com_proto_fsck.cc rename to console/commands/coms/unused/com_proto_fsck.cc diff --git a/console/commands/com_proto_group.cc b/console/commands/coms/unused/com_proto_group.cc similarity index 100% rename from console/commands/com_proto_group.cc rename to console/commands/coms/unused/com_proto_group.cc diff --git a/console/commands/com_proto_io.cc b/console/commands/coms/unused/com_proto_io.cc similarity index 100% rename from console/commands/com_proto_io.cc rename to console/commands/coms/unused/com_proto_io.cc diff --git a/console/commands/com_proto_node.cc b/console/commands/coms/unused/com_proto_node.cc similarity index 100% rename from console/commands/com_proto_node.cc rename to console/commands/coms/unused/com_proto_node.cc diff --git a/console/commands/com_proto_ns.cc b/console/commands/coms/unused/com_proto_ns.cc similarity index 100% rename from console/commands/com_proto_ns.cc rename to console/commands/coms/unused/com_proto_ns.cc diff --git a/console/commands/com_proto_quota.cc b/console/commands/coms/unused/com_proto_quota.cc similarity index 100% rename from console/commands/com_proto_quota.cc rename to console/commands/coms/unused/com_proto_quota.cc diff --git a/console/commands/com_proto_recycle.cc b/console/commands/coms/unused/com_proto_recycle.cc similarity index 100% rename from console/commands/com_proto_recycle.cc rename to console/commands/coms/unused/com_proto_recycle.cc diff --git a/console/commands/com_proto_register.cc b/console/commands/coms/unused/com_proto_register.cc similarity index 98% rename from console/commands/com_proto_register.cc rename to console/commands/coms/unused/com_proto_register.cc index 7ecb34956d..d376fe5c95 100644 --- a/console/commands/com_proto_register.cc +++ b/console/commands/coms/unused/com_proto_register.cc @@ -122,7 +122,7 @@ RegisterHelper::ParseCommand(const char* arg) } reg->mutable_ctime()->set_sec(ts.tv_sec); reg->mutable_ctime()->set_nsec(ts.tv_nsec); - } else if (param.substr(0,6) == "atime=") { + } else if (param.substr(0,6) == "atime=") { std::string t = param.substr(6); struct timespec ts; if (eos::common::Timing::Timespec_from_TimespecStr(t,ts)) { @@ -130,7 +130,7 @@ RegisterHelper::ParseCommand(const char* arg) } reg->mutable_atime()->set_sec(ts.tv_sec); reg->mutable_atime()->set_nsec(ts.tv_nsec); - } else if (param.substr(0,13) == "atimeifnewer=") { + } else if (param.substr(0,13) == "atimeifnewer=") { std::string t = param.substr(13); struct timespec ts; if (eos::common::Timing::Timespec_from_TimespecStr(t,ts)) { @@ -178,7 +178,7 @@ RegisterHelper::ParseCommand(const char* arg) return false; } reg->set_path(path); - } + } } else { break; } diff --git a/console/commands/com_proto_rm.cc b/console/commands/coms/unused/com_proto_rm.cc similarity index 100% rename from console/commands/com_proto_rm.cc rename to console/commands/coms/unused/com_proto_rm.cc diff --git a/console/commands/com_proto_route.cc b/console/commands/coms/unused/com_proto_route.cc similarity index 100% rename from console/commands/com_proto_route.cc rename to console/commands/coms/unused/com_proto_route.cc diff --git a/console/commands/com_proto_sched.cc b/console/commands/coms/unused/com_proto_sched.cc similarity index 100% rename from console/commands/com_proto_sched.cc rename to console/commands/coms/unused/com_proto_sched.cc diff --git a/console/commands/com_proto_space.cc b/console/commands/coms/unused/com_proto_space.cc similarity index 100% rename from console/commands/com_proto_space.cc rename to console/commands/coms/unused/com_proto_space.cc diff --git a/console/commands/com_proto_token.cc b/console/commands/coms/unused/com_proto_token.cc similarity index 100% rename from console/commands/com_proto_token.cc rename to console/commands/coms/unused/com_proto_token.cc diff --git a/console/commands/com_pwd.cc b/console/commands/coms/unused/com_pwd.cc similarity index 100% rename from console/commands/com_pwd.cc rename to console/commands/coms/unused/com_pwd.cc diff --git a/console/commands/com_quit.cc b/console/commands/coms/unused/com_quit.cc similarity index 100% rename from console/commands/com_quit.cc rename to console/commands/coms/unused/com_quit.cc diff --git a/console/commands/com_quota.cc b/console/commands/coms/unused/com_quota.cc similarity index 99% rename from console/commands/com_quota.cc rename to console/commands/coms/unused/com_quota.cc index b28750efc9..19cb00cb40 100644 --- a/console/commands/com_quota.cc +++ b/console/commands/coms/unused/com_quota.cc @@ -309,12 +309,12 @@ com_quota(char* arg1) space.c_str()); fprintf(stdout, "Confirm the deletion by typing => "); XrdOucString confirmation = ""; - + for (int i = 0; i < 10; i++) { // coverity[DC.WEAK_CRYPTO] confirmation += (int)(9.0 * rand() / RAND_MAX); } - + fprintf(stdout, "%s\n", confirmation.c_str()); fprintf(stdout, " => "); getline(std::cin, s); diff --git a/console/commands/com_rclone.cc b/console/commands/coms/unused/com_rclone.cc similarity index 98% rename from console/commands/com_rclone.cc rename to console/commands/coms/unused/com_rclone.cc index 4e933126f4..d920c0dcf7 100644 --- a/console/commands/com_rclone.cc +++ b/console/commands/coms/unused/com_rclone.cc @@ -28,7 +28,6 @@ #include "common/Timing.hh" #include "common/Path.hh" #include "common/LayoutId.hh" -#include "common/CopyProcess.hh" /*----------------------------------------------------------------------------*/ #include @@ -555,7 +554,7 @@ int setDirMtime(const std::string& i, eos::common::Path& prefix, } } -eos::common::CopyProcess copyProcess; +XrdCl::CopyProcess copyProcess; std::vector tprops; XrdCl::PropertyList* copyFile(const std::string& i, eos::common::Path& src, @@ -593,7 +592,7 @@ XrdCl::PropertyList* copyFile(const std::string& i, eos::common::Path& src, result->Set("source", srcurl); result->Set("target", dsturl); - //props.Set("parallel", 64); + // props.Set("parallel", 10); if (verbose) { std::cout << "[ copy file ] : srcurl: " << srcurl << " dsturl: " << dsturl << std::endl; @@ -1354,34 +1353,15 @@ com_rclone(char* arg1) class RCloneProgressHandler : public XrdCl::CopyProgressHandler { public: - RCloneProgressHandler() { - ext_tot = 0; - bp = bt = 0; - n = 0; - } - virtual void BeginJob(uint16_t jobNum, uint16_t jobTotal, const URL* source, const URL* destination) { - if (!n) { - // do this only once - n = jobNum; - } - } - - void Output() { - if (verbose) { - std::cerr << "[ " << n << "/" << ext_tot << " ] files copied" << std::endl; - } else { - if (!is_silent) { - std::cerr << "[ " << n << "/" << ext_tot << " ] files copied" << "\r"; - } - } + n = jobNum; + tot = jobTotal; } - virtual void EndJob(uint16_t jobNum, const PropertyList* result) { @@ -1411,15 +1391,23 @@ com_rclone(char* arg1) } } } - n++; - Output(); }; virtual void JobProgress(uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal) { - // we don't want output here + bp = bytesProcessed; + bt = bytesTotal; + n = jobNum; + + if (verbose) { + std::cerr << "[ " << jobNum << "/" << tot << " ] files copied" << std::endl; + } else { + if (!is_silent) { + std::cerr << "[ " << jobNum << "/" << tot << " ] files copied" << "\r"; + } + } } virtual bool ShouldCancel(uint16_t jobNum) @@ -1431,21 +1419,15 @@ com_rclone(char* arg1) std::atomic bp; std::atomic bt; std::atomic n; - std::atomic ext_tot; + std::atomic tot; }; RCloneProgressHandler copyProgress; - copyProgress.ext_tot = copyProcess.Jobs(); if (!is_silent && verbose) { std::cerr << "# preparing" << std::endl; } - - PropertyList processConfig; - processConfig.Set( "jobType", "configuration" ); - processConfig.Set( "parallel", 64 ); - copyProcess.AddJob( processConfig, 0 ); - + copyProcess.Prepare(); if (!is_silent && verbose) { diff --git a/console/commands/com_reconnect.cc b/console/commands/coms/unused/com_reconnect.cc similarity index 100% rename from console/commands/com_reconnect.cc rename to console/commands/coms/unused/com_reconnect.cc diff --git a/console/commands/com_report.cc b/console/commands/coms/unused/com_report.cc similarity index 100% rename from console/commands/com_report.cc rename to console/commands/coms/unused/com_report.cc diff --git a/console/commands/com_rm.cc b/console/commands/coms/unused/com_rm.cc similarity index 99% rename from console/commands/com_rm.cc rename to console/commands/coms/unused/com_rm.cc index 7abf25862e..b4a1bf6e3b 100644 --- a/console/commands/com_rm.cc +++ b/console/commands/coms/unused/com_rm.cc @@ -41,7 +41,7 @@ com_rm(char* arg1) eos::common::Path* cPath = 0; XrdOucString in = "mgm.cmd=rm&"; bool noconfirmation = false; - + if (wants_help(arg1)) { goto com_rm_usage; } @@ -69,7 +69,7 @@ com_rm(char* arg1) fprintf(stderr,"disabling configmration\n"); noconfirmation=true; } - + do { XrdOucString param = subtokenizer.GetToken(); if (param.length()) { diff --git a/console/commands/com_rmdir.cc b/console/commands/coms/unused/com_rmdir.cc similarity index 100% rename from console/commands/com_rmdir.cc rename to console/commands/coms/unused/com_rmdir.cc diff --git a/console/commands/com_role.cc b/console/commands/coms/unused/com_role.cc similarity index 100% rename from console/commands/com_role.cc rename to console/commands/coms/unused/com_role.cc diff --git a/console/commands/com_rtlog.cc b/console/commands/coms/unused/com_rtlog.cc similarity index 100% rename from console/commands/com_rtlog.cc rename to console/commands/coms/unused/com_rtlog.cc diff --git a/console/commands/com_scitoken.cc b/console/commands/coms/unused/com_scitoken.cc similarity index 100% rename from console/commands/com_scitoken.cc rename to console/commands/coms/unused/com_scitoken.cc diff --git a/console/commands/com_silent.cc b/console/commands/coms/unused/com_silent.cc similarity index 100% rename from console/commands/com_silent.cc rename to console/commands/coms/unused/com_silent.cc diff --git a/console/commands/com_squash.cc b/console/commands/coms/unused/com_squash.cc similarity index 100% rename from console/commands/com_squash.cc rename to console/commands/coms/unused/com_squash.cc diff --git a/console/commands/com_stat.cc b/console/commands/coms/unused/com_stat.cc similarity index 100% rename from console/commands/com_stat.cc rename to console/commands/coms/unused/com_stat.cc diff --git a/console/commands/com_status.cc b/console/commands/coms/unused/com_status.cc similarity index 100% rename from console/commands/com_status.cc rename to console/commands/coms/unused/com_status.cc diff --git a/console/commands/com_test.cc b/console/commands/coms/unused/com_test.cc similarity index 100% rename from console/commands/com_test.cc rename to console/commands/coms/unused/com_test.cc diff --git a/console/commands/com_timing.cc b/console/commands/coms/unused/com_timing.cc similarity index 100% rename from console/commands/com_timing.cc rename to console/commands/coms/unused/com_timing.cc diff --git a/console/commands/com_touch.cc b/console/commands/coms/unused/com_touch.cc similarity index 100% rename from console/commands/com_touch.cc rename to console/commands/coms/unused/com_touch.cc diff --git a/console/commands/com_tracker.cc b/console/commands/coms/unused/com_tracker.cc similarity index 100% rename from console/commands/com_tracker.cc rename to console/commands/coms/unused/com_tracker.cc diff --git a/console/commands/com_version.cc b/console/commands/coms/unused/com_version.cc similarity index 100% rename from console/commands/com_version.cc rename to console/commands/coms/unused/com_version.cc diff --git a/console/commands/com_vid.cc b/console/commands/coms/unused/com_vid.cc similarity index 100% rename from console/commands/com_vid.cc rename to console/commands/coms/unused/com_vid.cc diff --git a/console/commands/com_who.cc b/console/commands/coms/unused/com_who.cc similarity index 100% rename from console/commands/com_who.cc rename to console/commands/coms/unused/com_who.cc diff --git a/console/commands/com_whoami.cc b/console/commands/coms/unused/com_whoami.cc similarity index 100% rename from console/commands/com_whoami.cc rename to console/commands/coms/unused/com_whoami.cc diff --git a/console/commands/helpers/AclHelper.hh b/console/commands/helpers/AclHelper.hh index eee6521a5f..d868af203f 100644 --- a/console/commands/helpers/AclHelper.hh +++ b/console/commands/helpers/AclHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class AclHelper diff --git a/console/commands/helpers/FsHelper.hh b/console/commands/helpers/FsHelper.hh index 968195d210..8570dc3981 100644 --- a/console/commands/helpers/FsHelper.hh +++ b/console/commands/helpers/FsHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class FsHelper diff --git a/console/commands/helpers/FsckHelper.hh b/console/commands/helpers/FsckHelper.hh index a20e3cd266..9042a9c513 100644 --- a/console/commands/helpers/FsckHelper.hh +++ b/console/commands/helpers/FsckHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class FsckHelper diff --git a/console/commands/ICmdHelper.cc b/console/commands/helpers/ICmdHelper.cc similarity index 99% rename from console/commands/ICmdHelper.cc rename to console/commands/helpers/ICmdHelper.cc index a0c2627b45..1797f8545b 100644 --- a/console/commands/ICmdHelper.cc +++ b/console/commands/helpers/ICmdHelper.cc @@ -21,7 +21,7 @@ * along with this program. If not, see .* ************************************************************************/ -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" #include "common/Logging.hh" #include "common/SymKeys.hh" #include @@ -102,7 +102,7 @@ ICmdHelper::ExecuteWithoutPrint(bool add_route) if (getenv("EOSAPP")) { oss << "&eos.app=" << getenv("EOSAPP"); } - + if (mGlobalOpts.mDebug) { PrintDebugMsg(oss.str()); } diff --git a/console/commands/ICmdHelper.hh b/console/commands/helpers/ICmdHelper.hh similarity index 100% rename from console/commands/ICmdHelper.hh rename to console/commands/helpers/ICmdHelper.hh diff --git a/console/commands/helpers/NewfindHelper.hh b/console/commands/helpers/NewfindHelper.hh index d43806fbd7..b6e4c615dc 100644 --- a/console/commands/helpers/NewfindHelper.hh +++ b/console/commands/helpers/NewfindHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class NewfindHelper @@ -54,7 +54,7 @@ public: //! @return true if successful, otherwise false //---------------------------------------------------------------------------- bool ParseCommand(const char* arg) override; - + int FindXroot(std::string path); int FindAs3(std::string path); diff --git a/console/commands/helpers/NodeHelper.hh b/console/commands/helpers/NodeHelper.hh index ec3948080c..4aeb85b7e4 100644 --- a/console/commands/helpers/NodeHelper.hh +++ b/console/commands/helpers/NodeHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class NodeHelper diff --git a/console/commands/helpers/RecycleHelper.hh b/console/commands/helpers/RecycleHelper.hh index 158f713b49..a887ba593d 100644 --- a/console/commands/helpers/RecycleHelper.hh +++ b/console/commands/helpers/RecycleHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class RecycleHelper diff --git a/console/commands/helpers/TokenHelper.hh b/console/commands/helpers/TokenHelper.hh index 545c62d7d1..f001e47235 100644 --- a/console/commands/helpers/TokenHelper.hh +++ b/console/commands/helpers/TokenHelper.hh @@ -22,7 +22,7 @@ ************************************************************************/ #pragma once -#include "console/commands/ICmdHelper.hh" +#include "console/commands/helpers/ICmdHelper.hh" //------------------------------------------------------------------------------ //! Class TokenHelper diff --git a/console/commands/native/CoreNativeCommands.cc b/console/commands/native/CoreNativeCommands.cc new file mode 100644 index 0000000000..d84dcd6bb5 --- /dev/null +++ b/console/commands/native/CoreNativeCommands.cc @@ -0,0 +1,177 @@ +// ---------------------------------------------------------------------- +// File: CoreNativeCommands.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include + +namespace { +class HelpCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "help"; + } + const char* + description() const override + { + return "Display this text"; + } + bool + requiresMgm(const std::string&) const override + { + return false; + } + int + run(const std::vector& args, CommandContext&) override + { + if (args.empty()) { + auto& all = CommandRegistry::instance().all(); + fprintf(stderr, "Available commands:\n"); + // Make a copy and sort by command name + std::vector sorted(all.begin(), all.end()); + std::sort(sorted.begin(), sorted.end(), + [](const IConsoleCommand* a, const IConsoleCommand* b) { + return std::string(a->name()) < std::string(b->name()); + }); + for (auto* c : sorted) { + fprintf(stderr, " %-16s %s\n", c->name(), c->description()); + } + return 0; + } + // Print detailed help for a specific command + IConsoleCommand* cmd = CommandRegistry::instance().find(args[0].c_str()); + if (!cmd) { + fprintf(stderr, "error: unknown command '%s'\n", args[0].c_str()); + return (global_retc = EINVAL); + } + cmd->printHelp(); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "usage: help [command]\n"); + } +}; + +class ToggleFlagCommand : public IConsoleCommand { +public: + enum Which { JSON, SILENT, TIMING }; + ToggleFlagCommand(const char* n, const char* d, Which w) + : mName(n), mDesc(d), mWhich(w) + { + } + const char* + name() const override + { + return mName.c_str(); + } + const char* + description() const override + { + return mDesc.c_str(); + } + bool + requiresMgm(const std::string&) const override + { + return false; + } + int + run(const std::vector&, CommandContext&) override + { + switch (mWhich) { + case JSON: + ::json = (!::json); + gGlobalOpts.mJsonFormat = ::json; + if (::json) { + ::interactive = false; + ::global_highlighting = false; + } + if (!::silent) { + fprintf(stderr, "json=%d\n", ::json); + } + break; + case SILENT: + ::silent = (!::silent); + break; + case TIMING: + ::timing = (!::timing); + break; + } + return 0; + } + void + printHelp() const override + { + } + +private: + std::string mName; + std::string mDesc; + Which mWhich; +}; + +class QuitCommand : public IConsoleCommand { +public: + QuitCommand(const char* n) : mName(n) {} + const char* + name() const override + { + return mName.c_str(); + } + const char* + description() const override + { + return "Exit from EOS console"; + } + bool + requiresMgm(const std::string&) const override + { + return false; + } + int + run(const std::vector&, CommandContext&) override + { + ::done = 1; + return 0; + } + void + printHelp() const override + { + } + +private: + std::string mName; +}; +} // namespace + +void +RegisterCoreNativeCommands() +{ + CommandRegistry::instance().reg(std::make_unique()); + class HelpAlias : public HelpCommand { + public: + const char* + name() const override + { + return "?"; + } + }; + CommandRegistry::instance().reg(std::make_unique()); + CommandRegistry::instance().reg(std::make_unique( + "json", "Toggle JSON output flag for stdout", ToggleFlagCommand::JSON)); + CommandRegistry::instance().reg(std::make_unique( + "silent", "Toggle silent flag for stdout", ToggleFlagCommand::SILENT)); + CommandRegistry::instance().reg(std::make_unique( + "timing", "Toggle timing flag for execution time measurement", + ToggleFlagCommand::TIMING)); + CommandRegistry::instance().reg(std::make_unique("quit")); + CommandRegistry::instance().reg(std::make_unique("exit")); + CommandRegistry::instance().reg(std::make_unique(".q")); +} diff --git a/console/commands/native/LegacySymbols.cc b/console/commands/native/LegacySymbols.cc new file mode 100644 index 0000000000..63604d701d --- /dev/null +++ b/console/commands/native/LegacySymbols.cc @@ -0,0 +1,10 @@ +// ---------------------------------------------------------------------- +// File: LegacySymbols.cc +// Purpose: Provide legacy com_* symbol definitions (no registrations) +// ---------------------------------------------------------------------- + +// Include only legacy implementations still referenced by native wrappers +// Direct com_* dependencies from native commands +// com_cp/com_cat: provided by cp-cmd-native.cc +// com_rclone: provided by rclone-cmd-native.cc +// com_squash: provided by squash-cmd-native.cc (no external callers) diff --git a/console/commands/native/access-proto-native.cc b/console/commands/native/access-proto-native.cc new file mode 100644 index 0000000000..33a7916853 --- /dev/null +++ b/console/commands/native/access-proto-native.cc @@ -0,0 +1,354 @@ +// ---------------------------------------------------------------------- +// File: access-proto-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include +#include +#include + +namespace { +std::string MakeAccessHelp() +{ + std::ostringstream oss; + oss << "Usage: access ban|unban|allow|unallow|set|rm|ls [OPTIONS]\n" + "'[eos] access ..' provides the access interface of EOS to " + "allow/disallow hosts/domains and/or users\n\n" + "Subcommands:\n" + "access ban user|group|host|domain : ban user, " + "group, host or domain with identifier \n" + "\t : can be a user name, user id, group name, group id, " + "hostname or IP or domainname\n\n" + "access unban user|group|host|domain : unban user, " + "group, host or domain with identifier \n" + "\t : can be a user name, user id, group name, group id, " + "hostname or IP or domainname\n\n" + "access allow user|group|host|domain : allows this " + "user, group, host or domain access\n" + "\t : can be a user name, user id, group name, group id, " + "hostname or IP or domainname\n\n" + "access unallow user|group|host|domain : unallows " + "this user,group, host or domain access\n" + "\t : can be a user name, user id, group name, group id, " + "hostname or IP or domainname\n\n" + "\t HINT: if you add any 'allow' the instance allows only the listed " + "identity. A banned identifier will still overrule an allowed " + "identifier!\n\n" + "access set redirect [r|w|ENOENT|ENONET] : " + "allows to set a global redirection to \n" + "\t : hostname to which all requests get " + "redirected\n" + "\t [r|w] : optional set a redirect for read/write " + "requests seperatly\n" + "\t [ENOENT] : optional set a redirect if a file is not " + "existing\n" + "\t [ENONET] : optional set a redirect if a file is offline\n" + "\t can be structured like " + ":] where holds each request for a " + "given time before redirecting\n\n" + "access set stall [r|w|ENOENT|ENONET] : " + "allows to set a global stall time\n" + "\t : time in seconds after which clients should " + "rebounce\n" + "\t [r|w] : optional set stall time for read/write " + "requests seperatly\n" + "\t [ENOENT] : optional set stall time if a file is not " + "existing\n" + "\t [ENONET] : optional set stall time if a file is offline\n" + "\n" + "access set limit rate:{user,group}:{name}:\n" + "\t rate:{user:group}:{name}: : stall the defined user group " + "for 5s if the exceeds a frequency of in a 5s " + "interval\n" + "\t - the instantaneous rate can " + "exceed this value by 33%%\n" + "\t rate:user:*: : apply to all users based on " + "user counter\n" + "\t rate:group:*:: apply to all groups based on " + "group counter\n" + "\t set to 0 (zero) " + "to continuously stall the user or group\n\n" + "access set limit threads:{*,max,}\n" + "\t threads:max : set the maximum number of " + "threads running in parallel\n" + "\t threads:* : set the default thread pool " + "limit for each user\n" + "\t threads: : set a specific thread pool " + "limit for user \n\n" + "access set limit rate:user:{name}:FindFiles :\n\tset find " + "query limit to for user {name}\n\n" + "access set limit rate:user:{name}:FindDirs:\n\tset find query " + "limit to for user {name}\n\n" + "access set limit rate:group:{name}:FindFiles :\n\tset find " + "query limit to for group {name}\n\n" + "access set limit rate:group:{name}:FindDirs :\n\tset find " + "query limit to for group {name}\n\n" + "access set limit rate:user:*:FindFiles :\n\tset default find " + "query limit to for everybody\n\n" + "access set limit rate:user:*:FindDirs :\n\tset default find " + "query limit to for everybody\n\n" + "\t HINT : rule strength => user-limit >> group-limit >> " + "wildcard-limit\n\n" + "access rm redirect [r|w|ENOENT|ENONET] : removes global " + "redirection\n\n" + "access rm stall [r|w|ENOENT|ENONET] : removes global " + "stall time\n\n" + "access rm limit rate:{user,group}:{name}: : remove rate " + "limitation\n\n" + "access rm limit threads:{max,*,} : remove thread pool " + "limit\n\n" + "access ls [-m] [-n] : print banned,unbanned user,group, hosts\n" + "\t -m : output in monitoring format with =\n" + "\t -n : don't translate uid/gids to names\n\n" + "Examples:\n" + " access ban host foo : Ban host foo\n" + " access ban domain bar : Ban domain bar\n" + " access allow domain nobody@bar : Allows user nobody " + "from domain bar\n" + " access allow domain - : use domain allow as " + "whitelist - e.g. nobody@bar will additionally allow the nobody user " + "from domain bar!\n" + " access allow domain bar : Allow only domain " + "bar\n" + " access set redirect foo : Redirect all " + "requests to host foo\n" + " access set redirect foo:1094:1000 : Redirect all " + "requests to host foo:1094 and hold each reqeust for 1000ms\n" + " access rm redirect : Remove redirection " + "to previously defined host foo\n" + " access set stall 60 : Stall all clients " + "by 60 seconds\n" + " access ls : Print all defined " + "access rules\n" + " access set limit 100 rate:user:*:OpenRead : Limit the open for " + "read rate to a frequency of 100 Hz for all users\n" + " access set limit 0 rate:user:ab:OpenRead : Limit the open for " + "read rate for the ab user to 0 Hz, to continuously stall it\n" + " access set limit 2000 rate:group:zp:Stat : Limit the stat rate " + "for the zp group to 2kHz\n" + " access set limit 500 threads:* : Limit the thread " + "pool usage to 500 threads per user\n" + " access rm limit rate:user:*:OpenRead : Removes the defined " + "limit\n" + " access rm limit threads:* : Removes the default " + "per user thread pool limit\n" + " access stallhosts add stall foo*.bar : Add foo*.bar to the " + "list of hosts which are stalled by limit rules (white list)\n" + " access stallhosts remove stall foo*.bar : Remove foo*.bar " + "from the list of hosts which are stalled by limit rules (white list)\n" + " access stallhosts add nostall foo*.bar : Add foo*.bar to the " + "list of hosts which are never stalled by limit rules (black list)\n" + " access stallhosts remove nostall foo*.bar : Remove foo*.bar " + "from the list of hosts which are never stalled by limit rules (black " + "list)\n"; + return oss.str(); +} + +void ConfigureAccessApp(CLI::App& app) +{ + app.name("access"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeAccessHelp(); + })); +} + +class AccessProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "access"; + } + const char* + description() const override + { + return "Access Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + // access ban|unban|allow|unallow|set|rm|ls ... + if (args.empty() || wants_help(args[0].c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + const std::string& sub = args[0]; + XrdOucString in = "mgm.cmd=access"; + size_t i = 1; + auto next = [&](std::string& out) -> bool { + if (i < args.size()) { + out = args[i++]; + return true; + } + return false; + }; + + auto finish = [&]() { + global_retc = + ctx.outputResult(ctx.clientCommand(in, true, nullptr), true); + return 0; + }; + + if (sub == "ban" || sub == "unban" || sub == "allow" || sub == "unallow") { + in += "&mgm.subcmd="; + in += sub.c_str(); + std::string type, id; + if (!next(type) || !next(id)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (type == "host") { + in += "&mgm.access.host="; + in += id.c_str(); + } else if (type == "domain") { + in += "&mgm.access.domain="; + in += id.c_str(); + } else if (type == "user") { + in += "&mgm.access.user="; + in += id.c_str(); + } else if (type == "group") { + in += "&mgm.access.group="; + in += id.c_str(); + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + return finish(); + } + + if (sub == "ls") { + in += "&mgm.subcmd=ls"; + // options: -m, -n + CLI::App app; + app.set_help_flag(""); + bool opt_m = false; + bool opt_n = false; + app.add_flag("-m", opt_m, "monitor format"); + app.add_flag("-n", opt_n, "numeric ids"); + std::vector rest(args.begin() + 1, args.end()); + std::vector cli_args = rest; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + std::string option; + if (opt_m) + option += "m"; + if (opt_n) + option += "n"; + if (!option.empty()) { + in += "&mgm.access.option="; + in += option.c_str(); + } + return finish(); + } + + if (sub == "set" || sub == "rm") { + in += "&mgm.subcmd="; + in += sub.c_str(); + std::string type; + if (!next(type)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + std::string id; + bool has_id = next(id); + if (!has_id) { + if (sub == "rm") { + id = "dummy"; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + std::string rtype; + if (sub == "rm") { + if (has_id) { + rtype = id; + } + } else { + next(rtype); + } + + if (type == "redirect") { + in += "&mgm.access.redirect="; + in += id.c_str(); + } else if (type == "stall") { + in += "&mgm.access.stall="; + in += id.c_str(); + } else if (type == "limit") { + in += "&mgm.access.stall="; + in += id.c_str(); + if (rtype.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if ((rtype.rfind("rate:user:", 0) == 0 || + rtype.rfind("rate:group:", 0) == 0) && + (rtype.find(':', 11) != std::string::npos)) { + in += "&mgm.access.type="; + in += rtype.c_str(); + } else if (!rtype.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + return finish(); + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + + if (!rtype.empty()) { + if (rtype == "r" || rtype == "w" || rtype == "ENONET" || + rtype == "ENOENT") { + in += "&mgm.access.type="; + in += rtype.c_str(); + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + return finish(); + } + + printHelp(); + global_retc = EINVAL; + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureAccessApp(app); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterAccessProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/accounting-cmd-native.cc b/console/commands/native/accounting-cmd-native.cc new file mode 100644 index 0000000000..8119702177 --- /dev/null +++ b/console/commands/native/accounting-cmd-native.cc @@ -0,0 +1,132 @@ +// ---------------------------------------------------------------------- +// File: accounting-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include +#include +#include + +namespace { +std::string MakeAccountingHelp() +{ + std::ostringstream oss; + oss << "Usage: accounting report [-f]\n" + << " accounting config -e [] -i []\n\n" + << " report prints accounting report in JSON, data is served from " + "cache if possible\n" + << " -f force synchronous report instead of using cache\n\n" + << " config configure caching behaviour\n" + << " -e expiry time in minutes (default 10)\n" + << " -i invalidity time in minutes, must be > expiry\n"; + return oss.str(); +} + +void ConfigureAccountingApp(CLI::App& app) +{ + app.name("accounting"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeAccountingHelp(); + })); +} + +class AccountingCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "accounting"; + } + const char* + description() const override + { + return "Accounting tools"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + if (args.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + const std::string& sub = args[0]; + XrdOucString in = "mgm.cmd=accounting"; + if (sub == "report") { + in += "&mgm.subcmd=report"; + CLI::App app; + app.set_help_flag(""); + app.allow_extras(); + bool opt_f = false; + app.add_flag("-f", opt_f, "force synchronous report"); + std::vector rest(args.begin() + 1, args.end()); + std::vector cli_args = rest; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (opt_f) { + in += "&mgm.option=f"; + } + } else if (sub == "config") { + in += "&mgm.subcmd=config"; + // -e -i + std::vector rest(args.begin() + 1, args.end()); + for (size_t i = 0; i < rest.size();) { + const std::string& tok = rest[i]; + if (tok == "-e" || tok == "-i") { + if (i + 1 >= rest.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + const std::string& val = rest[i + 1]; + if (tok == "-e") + in += "&mgm.accounting.expired="; + else + in += "&mgm.accounting.invalid="; + in += val.c_str(); + i += 2; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureAccountingApp(app); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterAccountingNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/acl-proto-native.cc b/console/commands/native/acl-proto-native.cc new file mode 100644 index 0000000000..738867fc5b --- /dev/null +++ b/console/commands/native/acl-proto-native.cc @@ -0,0 +1,101 @@ +// ---------------------------------------------------------------------- +// File: acl-proto-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/commands/helpers/AclHelper.hh" +#include +#include +#include + +namespace { +std::string MakeAclHelp() +{ + return "Usage: acl [-l|--list] [-R|--recursive] [-p|--position ] " + "[-f|--front] [--sys|--user] [] \n\n" + "Atomically set and modify ACLs for the given directory path/sub-tree.\n\n" + "Options:\n" + " -l, --list list ACL rules\n" + " -R, --recursive apply to directories recursively\n" + " -p, --position add the acl rule at specified position\n" + " -f, --front add the acl rule at the front position\n" + " --user handle user.acl rules on directory\n" + " --sys handle sys.acl rules on directory (admin only)\n\n" + " can be |cid:|cxid:\n\n" + " format: [u|g|egroup]: or [u|g|egroup]= with permissions.\n" + " \":\" modifies, \"=\" sets. Use + and - to add/remove flags.\n\n" + "Examples:\n" + " acl --user u:1001=rwx /eos/dev/\n" + " acl --user u:1001:-w /eos/dev\n" + " acl --front --user u:1001=rwx /eos/dev\n"; +} + +void ConfigureAclApp(CLI::App& app) +{ + app.name("acl"); + app.description("Acl Interface"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeAclHelp(); + })); +} + +class AclProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "acl"; + } + const char* + description() const override + { + return "Acl Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + (void)ctx; + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + AclHelper acl(gGlobalOpts); + if (!acl.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = acl.Execute(true, true); + return global_retc; + } + void + printHelp() const override + { + CLI::App app; + ConfigureAclApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterAclProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/archive-cmd-native.cc b/console/commands/native/archive-cmd-native.cc new file mode 100644 index 0000000000..a07d31db30 --- /dev/null +++ b/console/commands/native/archive-cmd-native.cc @@ -0,0 +1,172 @@ +// ---------------------------------------------------------------------- +// File: archive-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include +#include + +namespace { +std::string MakeArchiveHelp() +{ + std::ostringstream oss; + oss << "Usage: archive [args...]\n\n" + << "Subcommands:\n" + << " create create " + "archive file\n" + << " put [--retry] copy files from EOS to " + "archive location\n" + << " get [--retry] recall archive back to " + "EOS\n" + << " purge [--retry] purge files on disk\n" + << " transfers [all|put|get|purge|job_uuid] show status of running " + "jobs\n" + << " list [] show status of archived " + "directories in subtree\n" + << " kill kill transfer\n" + << " delete delete files from tape, " + "keeping the ones on disk\n" + << " help [--help|-h] display help message\n"; + return oss.str(); +} + +void ConfigureArchiveApp(CLI::App& app, + std::string& subcmd, + bool& retry, + std::string& arg) +{ + app.name("archive"); + app.description("Archive Interface"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeArchiveHelp(); + })); + app.add_option("subcmd", subcmd, + "create|put|get|purge|delete|transfers|list|kill|help") + ->required(); + app.add_flag("--retry", retry, "retry on failure"); + app.add_option("arg", arg, "path, job_uuid, or option"); +} + +class ArchiveCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "archive"; + } + const char* + description() const override + { + return "Archive Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + std::string subcmd; + bool retry = false; + std::string arg; + ConfigureArchiveApp(app, subcmd, retry, arg); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::ostringstream in_cmd; + in_cmd << "mgm.cmd=archive&mgm.subcmd=" << subcmd; + + if (subcmd == "create") { + std::string p = arg.empty() ? gPwd.c_str() : arg; + XrdOucString ap = abspath(p.c_str()); + in_cmd << "&mgm.archive.path=" << ap.c_str(); + } else if (subcmd == "put" || subcmd == "get" || subcmd == "purge" || + subcmd == "delete") { + std::string p = arg.empty() ? gPwd.c_str() : arg; + if (retry) + in_cmd << "&mgm.archive.option=r"; + XrdOucString ap = abspath(p.c_str()); + in_cmd << "&mgm.archive.path=" << ap.c_str(); + } else if (subcmd == "transfers") { + if (arg.empty()) + in_cmd << "&mgm.archive.option=all"; + else + in_cmd << "&mgm.archive.option=" << arg; + } else if (subcmd == "list") { + if (arg.empty()) + in_cmd << "&mgm.archive.path=/"; + else if (arg == "./" || arg == ".") { + XrdOucString ap = abspath(gPwd.c_str()); + in_cmd << "&mgm.archive.path=" << ap.c_str(); + } else + in_cmd << "&mgm.archive.path=" << arg; + } else if (subcmd == "kill") { + if (arg.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in_cmd << "&mgm.archive.option=" << arg; + } else if (subcmd == "help") { + printHelp(); + return 0; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + + XrdOucString in = in_cmd.str().c_str(); + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + std::string subcmd; + bool retry = false; + std::string arg; + ConfigureArchiveApp(app, subcmd, retry, arg); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterArchiveNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/attr-cmd-native.cc b/console/commands/native/attr-cmd-native.cc new file mode 100644 index 0000000000..b9459dcffe --- /dev/null +++ b/console/commands/native/attr-cmd-native.cc @@ -0,0 +1,548 @@ +// ---------------------------------------------------------------------- +// File: attr-native.cc +// ---------------------------------------------------------------------- + +#include "common/SymKeys.hh" +#include "common/Utils.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::string MakeAttrHelp() +{ + std::ostringstream oss; + oss << "Usage: attr [OPTIONS] ls|set|get|rm ...\n\n"; + oss << "'[eos] attr ..' provides the extended attribute interface for " + "directories in EOS.\n\n"; + oss << "Options:\n" + << " attr [-r] ls \n" + << " List attributes of path\n" + << " -r : list recursive on all directory children\n" + << " attr [-r] set [-c] = \n" + << " Use = to set an empty value (e.g. sys.acl=)\n" + << " Set attributes of path (-r recursive, -c only if absent)\n" + << " attr [-r] set default=replica|raiddp|raid5|raid6|archive|qrain " + "\n" + << " Set EOS default layout attributes for the path\n" + << " attr [-r] [-V] get \n" + << " Get attributes of path (-r recursive, -V only print value)\n" + << " attr [-r] rm \n" + << " Delete attributes of path (-r recursive)\n" + << " attr [-r] link \n" + << " Link attributes of under (-r recursive)\n" + << " attr [-r] unlink \n" + << " Remove attribute link of (-r recursive)\n" + << " attr [-r] fold \n" + << " Fold attributes of when attr link is defined\n" + << " (identical attributes are removed locally)\n" + << "\n" + << "Remarks:\n"; + oss << " = " + "|fid:|fxid:|cid:|cxid:\n" + << " deprecated pid:|pxid:\n"; + oss << " If starts with 'sys.' you have to be member of the " + "sudoers group to see these attributes or modify.\n"; + oss << "\nAdministrator Variables:\n"; + oss << " sys.forced.space= : enforces to use " + " [configuration dependent]\n"; + oss << " sys.forced.group= : enforces to use " + ", where is the numerical index of . " + "[configuration dependent]\n"; + oss << " sys.forced.layout= : enforces to use " + " [=(plain,replica,raid5,raid6,archive,qrain)]\n"; + oss << " sys.forced.checksum= : enforces to use " + "file-level checksum \n"; + oss << " = " + "adler,crc32,crc32c,md5,sha\n"; + oss << " sys.forced.blockchecksum= : enforces to use " + "block-level checksum \n"; + oss << " = " + "adler,crc32,crc32c,md5,sha\n"; + oss << " sys.forced.nstripes= : enforces to use " + "stripes[= 1..16]\n"; + oss << " sys.forced.blocksize= : enforces to use a " + "blocksize of - can be 4k,64k,128k,256k or 1M \n"; + oss << " sys.forced.placementpolicy=[:geotag] : enforces to " + "use replica/stripe placement policy [=" + "{scattered|hybrid:|gathered:}]\n"; + oss << " sys.forced.nouserplacementpolicy=1 : disables user " + "defined replica/stripe placement policy\n"; + oss << " sys.forced.nouserlayout=1 : disables the user " + "settings with user.forced.\n"; + oss << " sys.forced.nofsselection=1 : disables user " + "defined filesystem selection with environment variables for reads\n"; + oss << " sys.forced.bookingsize= : set's the number of " + "bytes which get for each new created replica\n"; + oss << " sys.forced.minsize= : set's the minimum " + "number of bytes a file to be stored must have\n"; + oss << " sys.forced.maxsize= : set's the maximum " + "number of bytes a file to be stored can have\n"; + oss << " sys.forced.atomic=1 : if present enforce " + "atomic uploads e.g. files appear only when their upload is complete - " + "during the upload they have the name /..\n"; + oss << " sys.forced.leasetime=86400 : allows to overwrite " + "the eosxd client provided leasetime with a new value\n"; + oss << " sys.forced.iotype=direct|sync|dsync|csync" + << " : force the given " + "iotype for that directory\n"; + oss << " sys.mtime.propagation=1 : if present a change " + "under this directory propagates an mtime change up to all parents " + "until the attribute is not present anymore\n"; + oss << " sys.allow.oc.sync=1 : if present, " + "OwnCloud clients can sync pointing to this subtree\n"; + oss << "\n"; + oss << " sys.lru.expire.empty= : delete empty " + "directories older than \n"; + oss << " sys.lru.expire.match=[match1:,match2:..]\n"; + oss << " : defines the rule " + "that files with a given match will be removed if \n"; + oss << " they haven't been " + "accessed longer than ago. is defined like " + "3600,3600s,60min,1h,1mo,1y...\n"; + oss << " sys.lru.lowwatermark=\n"; + oss << " sys.lru.highwatermark= : if the watermark " + "reaches more than %%, files will be removed until the usage is " + "reaching %%.\n"; + oss << "\n"; + oss << " sys.lru.convert.match=[match1:,match2:,...]\n"; + oss << " defines the rule " + "that files with a given match will be converted to the layouts " + "defined by sys.conversion. when their access time reaches " + ".\n"; + oss << "\n"; + oss << " sys.stall.unavailable= : stall clients for " + " seconds if a needed file system is unavailable\n"; + oss << " sys.redirect.enoent= : redirect clients " + "opening non existing files to \n"; + oss << " sys.redirect.enonet= : redirect clients " + "opening inaccessible files to \n"; + oss << " sys.recycle=.... : define the recycle " + "bin - WARNING: use the 'recycle' interface\n"; + oss << " sys.recycle.keeptime= : define the time how " + "long files stay in a recycle bin\n"; + oss << " sys.recycle.keepratio=< 0 .. 1.0 > : ratio of used/max " + "quota for space and inodes in the recycle bin\n"; + oss << " sys.versioning= : keep versions " + "of a file\n"; + oss << " sys.acl= : set's an ACL\n"; + oss << " sys.eval.useracl : enables the " + "evaluation of user acls\n"; + oss << " sys.mask : masks all unix " + "access permissions\n"; + oss << " sys.owner.auth= : set's additional " + "owner on a directory\n"; + oss << " sys.attr.link= : symbolic links for " + "attributes\n"; + oss << " sys.http.index= : show a static page " + "as directory index\n"; + oss << " sys.accounting.*= : set accounting " + "attributes\n"; + oss << " sys.proc= : run arbitrary " + "command on accessing the file\n"; + oss << "\nUser Variables:\n"; + oss << " user.forced.space, user.forced.layout, user.forced.checksum, " + "etc. (s.a. Administrator Variables)\n"; + oss << "\nExamples:\n"; + oss << " attr set default=replica /eos/instance/2-replica\n"; + oss << " attr set sys.forced.nstripes=10 /eos/instance/archive\n"; + oss << " attr set sys.acl=g:xx::!d!u /eos/instance/no-update-deletion\n"; + oss << " attr set sys.forced.atomic=1 /eos/dev/instance/atomic/\n"; + oss << " attr set sys.attr.link=/eos/dev/origin-attr/ " + "/eos/dev/instance/attr-linked/\n"; + return oss.str(); +} + +void ConfigureAttrApp(CLI::App& app, + bool& opt_r, + bool& opt_V, + std::string& subcmd) +{ + app.name("attr"); + app.description("Attribute Interface"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeAttrHelp(); + })); + app.add_flag("-r", opt_r, "recursive"); + app.add_flag("-V", opt_V, "only print value (for get)"); + app.add_option("subcmd", subcmd, "ls|set|get|rm|link|unlink|fold") + ->required(); +} + +class AttrCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "attr"; + } + const char* + description() const override + { + return "Attribute Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + bool opt_r = false; + bool opt_V = false; + std::string sub; + ConfigureAttrApp(app, opt_r, opt_V, sub); + + // Insert "--" before first non-flag so CLI11 treats key=value (e.g. default=replica) + // as positional, not as --default=replica option + std::vector cli_args; + cli_args.reserve(args.size() + 1); + size_t i = 0; + while (i < args.size() && (args[i] == "-r" || args[i] == "-V")) { + cli_args.push_back(args[i]); + ++i; + } + cli_args.push_back("--"); + for (; i < args.size(); ++i) { + cli_args.push_back(args[i]); + } + // CLI11 parses from the back; reverse so "--" is seen first, then set, default=replica, path + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::string optionStr; + if (opt_r) + optionStr += 'r'; + if (opt_V) + optionStr += 'V'; + + std::vector remaining = app.remaining(); + + size_t idx = 0; + std::string arg; + + auto appendOption = [&](XrdOucString& target, const std::string& opt) { + if (!opt.empty()) { + target += "&mgm.option="; + target += opt.c_str(); + } + }; + + auto send_set = [&](const std::string& key, + const std::string& value, + const std::string& path, + bool conditional) -> int { + std::string opt = optionStr; + if (conditional && opt.find('c') == std::string::npos) { + opt.push_back('c'); + } + XrdOucString cmd = "mgm.cmd=attr&mgm.enc=b64"; + appendOption(cmd, opt); + XrdOucString k = key.c_str(); + XrdOucString v = value.c_str(); + if (key != "default" && key != "sys.attr.link") { + XrdOucString v64; + eos::common::SymKey::Base64(v, v64); + v = v64; + } + XrdOucString p = PathIdentifier(path.c_str(), true).c_str(); + cmd += "&mgm.subcmd=set&mgm.attr.key="; + cmd += k; + cmd += "&mgm.attr.value="; + cmd += v; + cmd += "&mgm.path="; + cmd += p; + return ctx.outputResult(ctx.clientCommand(cmd, false, nullptr), true); + }; + + // Skip "--" which CLI11 adds to remaining when used as positional separator + while (idx < remaining.size() && remaining[idx] == "--") { + ++idx; + } + if (idx >= remaining.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + arg = remaining[idx++]; + if (sub == "set" && arg == "-c") { + if (optionStr.find('c') == std::string::npos) { + optionStr.push_back('c'); + } + if (idx >= remaining.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + arg = remaining[idx++]; + } + + XrdOucString in = "mgm.cmd=attr&mgm.enc=b64"; + appendOption(in, optionStr); + + if (sub.empty() || arg.empty() || + (sub != "ls" && sub != "set" && sub != "get" && sub != "rm" && + sub != "link" && sub != "unlink" && sub != "fold")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + if (sub == "ls") { + XrdOucString path = PathIdentifier(arg.c_str(), true).c_str(); + in += "&mgm.subcmd=ls&mgm.path="; + in += path; + } else if (sub == "set" || sub == "link") { + std::string key = arg; + std::string value; + int epos = XrdOucString(key.c_str()).find("="); + const bool has_equals = (sub == "set" && epos != STR_NPOS); + if (sub == "link") { + key = "sys.attr.link"; + value = arg; + } else if (epos != STR_NPOS) { + value = key.substr(epos + 1); + key.erase(epos); + } else { + value.clear(); + } + + if (!value.empty() && value.rfind("\"", 0) == 0 && + value.size() >= 1 && value.back() != '"') { + while (idx < remaining.size()) { + value += " "; + value += remaining[idx]; + if (!remaining[idx].empty() && remaining[idx].back() == '"') { + ++idx; + break; + } + ++idx; + } + } + + if (value.empty() && !has_equals) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::string path; + if (sub == "link") { + if (idx >= remaining.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + path = remaining[idx++]; + } else { + if (idx >= remaining.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + path = remaining[idx++]; + } + + if (path.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + if (key == "default") { + bool conditional = optionStr.find('c') != std::string::npos; + std::vector> defaults; + if (value == "replica") { + defaults = {{"sys.forced.blocksize", "4k"}, + {"sys.forced.checksum", "adler"}, + {"sys.forced.layout", "replica"}, + {"sys.forced.nstripes", "2"}, + {"sys.forced.space", "default"}}; + } else if (value == "raiddp") { + defaults = {{"sys.forced.blocksize", "1M"}, + {"sys.forced.checksum", "adler"}, + {"sys.forced.layout", "raiddp"}, + {"sys.forced.nstripes", "6"}, + {"sys.forced.space", "default"}, + {"sys.forced.blockchecksum", "crc32c"}}; + } else if (value == "raid5") { + defaults = {{"sys.forced.blocksize", "1M"}, + {"sys.forced.checksum", "adler"}, + {"sys.forced.layout", "raid5"}, + {"sys.forced.nstripes", "5"}, + {"sys.forced.space", "default"}, + {"sys.forced.blockchecksum", "crc32c"}}; + } else if (value == "raid6") { + defaults = {{"sys.forced.blocksize", "1M"}, + {"sys.forced.checksum", "adler"}, + {"sys.forced.layout", "raid6"}, + {"sys.forced.nstripes", "6"}, + {"sys.forced.space", "default"}, + {"sys.forced.blockchecksum", "crc32c"}}; + } else if (value == "archive") { + defaults = {{"sys.forced.blocksize", "1M"}, + {"sys.forced.checksum", "adler"}, + {"sys.forced.layout", "archive"}, + {"sys.forced.nstripes", "8"}, + {"sys.forced.space", "default"}, + {"sys.forced.blockchecksum", "crc32c"}}; + } else if (value == "qrain") { + defaults = {{"sys.forced.blocksize", "1M"}, + {"sys.forced.checksum", "adler"}, + {"sys.forced.layout", "qrain"}, + {"sys.forced.nstripes", "12"}, + {"sys.forced.space", "default"}, + {"sys.forced.blockchecksum", "crc32c"}}; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + + int retc = 0; + for (const auto& entry : defaults) { + retc = retc || send_set(entry.first, entry.second, path, conditional); + } + global_retc = retc; + return 0; + } + + std::string encodedValue = value; + if (key != "default" && key != "sys.attr.link") { + XrdOucString in = value.c_str(); + XrdOucString v64; + eos::common::SymKey::Base64(in, v64); + encodedValue = v64.c_str(); + } + + if (sub == "set" && + XrdOucString(key.c_str()).endswith(".forced.placementpolicy")) { + XrdOucString in = encodedValue.c_str(); + XrdOucString ouc_policy; + eos::common::SymKey::DeBase64(in, ouc_policy); + std::string policy = ouc_policy.c_str(); + if (policy != "scattered" && + policy.rfind("hybrid:", 0) != 0 && + policy.rfind("gathered:", 0) != 0) { + fprintf(stderr, "Error: placement policy '%s' is invalid\n", + policy.c_str()); + global_retc = EINVAL; + return 0; + } + if (policy != "scattered") { + std::string targetgeotag = policy.substr(policy.find(':') + 1); + std::string tmp_geotag = eos::common::SanitizeGeoTag(targetgeotag); + if (tmp_geotag != targetgeotag) { + fprintf(stderr, "%s\n", tmp_geotag.c_str()); + global_retc = EINVAL; + return 0; + } + } + } + + XrdOucString k = key.c_str(); + XrdOucString v = encodedValue.c_str(); + XrdOucString p = PathIdentifier(path.c_str(), true).c_str(); + in += "&mgm.subcmd=set&mgm.attr.key="; + in += k; + in += "&mgm.attr.value="; + in += v; + in += "&mgm.path="; + in += p; + } else if (sub == "get") { + std::string key = arg; + if (idx >= remaining.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + std::string path = remaining[idx++]; + XrdOucString p = PathIdentifier(path.c_str(), true).c_str(); + in += "&mgm.subcmd=get&mgm.attr.key="; + in += key.c_str(); + in += "&mgm.path="; + in += p; + } else if (sub == "fold") { + std::string path = arg; + XrdOucString p = PathIdentifier(path.c_str(), true).c_str(); + in += "&mgm.subcmd=fold&mgm.path="; + in += p; + } else if (sub == "rm" || sub == "unlink") { + std::string key = arg; + std::string path; + if (sub == "unlink") { + key = "sys.attr.link"; + path = arg; + } else { + if (idx >= remaining.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + path = remaining[idx++]; + } + if (key.empty() || path.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = PathIdentifier(path.c_str(), true).c_str(); + in += "&mgm.subcmd=rm&mgm.attr.key="; + in += key.c_str(); + in += "&mgm.path="; + in += p; + } + + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + bool opt_r = false; + bool opt_V = false; + std::string subcmd; + ConfigureAttrApp(app, opt_r, opt_V, subcmd); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterAttrNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/backup-cmd-native.cc b/console/commands/native/backup-cmd-native.cc new file mode 100644 index 0000000000..4827451062 --- /dev/null +++ b/console/commands/native/backup-cmd-native.cc @@ -0,0 +1,153 @@ +// ---------------------------------------------------------------------- +// File: backup-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include +#include "console/ConsoleMain.hh" +#include +#include +#include + +namespace { +std::string MakeBackupHelp() +{ + std::ostringstream oss; + oss << "Usage: backup [options]\n\n" + << "Options:\n" + << " --ctime, --mtime s|m|h|d use the specified timewindow to " + "select entries for backup\n" + << " --excl_xattr val_1[,val_2]... extended attributes which are not " + "enforced and not checked during verification\n"; + return oss.str(); +} + +void ConfigureBackupApp(CLI::App& app, + std::string& ctime, + std::string& mtime, + std::string& excl_xattr) +{ + app.name("backup"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeBackupHelp(); + })); + app.add_option("--ctime", ctime, "ctime window"); + app.add_option("--mtime", mtime, "mtime window"); + app.add_option("--excl_xattr", excl_xattr, "exclude xattrs"); +} + +class BackupCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "backup"; + } + const char* + description() const override + { + return "Backup Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + if (args.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + const std::string& src = args[0]; + const std::string& dst = args[1]; + std::ostringstream in_cmd; + in_cmd << "mgm.cmd=backup&mgm.backup.src=" << src + << "&mgm.backup.dst=" << dst; + // parse optional flags + CLI::App app; + app.allow_extras(); + std::string ctime; + std::string mtime; + std::string excl_xattr; + ConfigureBackupApp(app, ctime, mtime, excl_xattr); + std::vector rest; + if (args.size() > 2) + rest.assign(args.begin() + 2, args.end()); + std::vector cli_args = rest; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + auto append_window = [&](const char* key, const std::string& val) { + if (val.empty()) + return; + char last = val.back(); + long seconds = 0; + if (last == 's') + seconds = 1; + else if (last == 'm') + seconds = 60; + else if (last == 'h') + seconds = 3600; + else if (last == 'd') + seconds = 24 * 3600; + else { + printHelp(); + global_retc = EINVAL; + return; + } + long v = strtol(val.c_str(), nullptr, 10); + if (v == 0L) { + printHelp(); + global_retc = EINVAL; + return; + } + struct timeval tv; + if (gettimeofday(&tv, NULL)) { + fprintf(stderr, "Error getting current timestamp\n"); + global_retc = EINVAL; + return; + } + in_cmd << "&mgm.backup.ttime=" << key + << "&mgm.backup.vtime=" << (tv.tv_sec - v * seconds); + }; + if (!ctime.empty()) + append_window("ctime", ctime); + if (!mtime.empty()) + append_window("mtime", mtime); + if (!excl_xattr.empty()) { + in_cmd << "&mgm.backup.excl_xattr=" << excl_xattr; + } + XrdOucString in = in_cmd.str().c_str(); + global_retc = ctx.outputResult(ctx.clientCommand(in, true, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + std::string ctime; + std::string mtime; + std::string excl_xattr; + ConfigureBackupApp(app, ctime, mtime, excl_xattr); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterBackupNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/cat-com-native.cc b/console/commands/native/cat-com-native.cc new file mode 100644 index 0000000000..13c7e0ae87 --- /dev/null +++ b/console/commands/native/cat-com-native.cc @@ -0,0 +1,54 @@ +// ---------------------------------------------------------------------- +// File: cat-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include + +extern int com_cat(char*); + +namespace { +class CatCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "cat"; + } + const char* + description() const override + { + return "Cat a file"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + return com_cat((char*)joined.c_str()); + } + void + printHelp() const override + { + fprintf(stderr, "Usage: cat \n"); + } +}; +} // namespace + +void +RegisterCatNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/cd-cmd-native.cc b/console/commands/native/cd-cmd-native.cc new file mode 100644 index 0000000000..905e2ae73c --- /dev/null +++ b/console/commands/native/cd-cmd-native.cc @@ -0,0 +1,157 @@ +// ---------------------------------------------------------------------- +// File: cd-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include + +namespace { +std::string MakeCdHelp() +{ + return "Usage: cd | cd - | cd ~\n"; +} + +void ConfigureCdApp(CLI::App& app, std::string& path) +{ + app.name("cd"); + app.description("Change directory"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeCdHelp(); + })); + app.add_option("path", path, "path (- for previous, ~ for home)") + ->default_val(""); +} + +class CdCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "cd"; + } + const char* + description() const override + { + return "Change directory"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + fprintf(stderr, "%s", MakeCdHelp().c_str()); + global_retc = EINVAL; + return 0; + } + + std::string path; + if (!args.empty()) { + CLI::App app; + ConfigureCdApp(app, path); + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + fprintf(stderr, "%s", MakeCdHelp().c_str()); + global_retc = EINVAL; + return 0; + } + } + + static XrdOucString opwd = "/"; + static XrdOucString oopwd = "/"; + + XrdOucString lsminuss; + XrdOucString newpath; + XrdOucString oldpwd; + + XrdOucString arg = path.c_str(); + + if (arg == "-") { + oopwd = opwd; + arg = (char*)opwd.c_str(); + } + + opwd = gPwd; + newpath = abspath(arg.c_str()); + oldpwd = gPwd; + + if ((arg == "") || (arg == "~")) { + if (getenv("EOS_HOME")) { + newpath = abspath(getenv("EOS_HOME")); + } else { + fprintf(stderr, + "warning: there is no home directory defined via EOS_HOME\n"); + newpath = opwd; + } + } + + gPwd = newpath; + if ((!gPwd.endswith("/")) && (!gPwd.endswith("/\""))) { + gPwd += "/"; + } + + while (gPwd.replace("/./", "/")) { + } + + int dppos = 0; + while ((dppos = gPwd.find("/../")) != STR_NPOS) { + if (dppos == 0) { + gPwd = oldpwd; + break; + } + int rpos = gPwd.rfind("/", dppos - 1); + if (rpos != STR_NPOS) { + gPwd.erase(rpos, dppos - rpos + 3); + } else { + gPwd = oldpwd; + break; + } + } + + if ((!gPwd.endswith("/")) && (!gPwd.endswith("/\""))) { + gPwd += "/"; + } + + lsminuss = "mgm.cmd=cd&mgm.path="; + lsminuss += gPwd; + lsminuss += "&mgm.option=s"; + global_retc = + ctx.outputResult(ctx.clientCommand(lsminuss, false, nullptr), true); + if (global_retc) { + gPwd = oldpwd; + } + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeCdHelp().c_str()); + } +}; +} // namespace + +void +RegisterCdNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/chmod-cmd-native.cc b/console/commands/native/chmod-cmd-native.cc new file mode 100644 index 0000000000..1b2f4e5750 --- /dev/null +++ b/console/commands/native/chmod-cmd-native.cc @@ -0,0 +1,118 @@ +// ---------------------------------------------------------------------- +// File: chmod-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include + +namespace { +std::string MakeChmodHelp() +{ + std::ostringstream oss; + oss << "Usage: chmod [-r] : set " + "mode for (-r recursive)\n"; + oss << " can be only numerical like 755, 644, 700\n"; + return oss.str(); +} + +void ConfigureChmodApp(CLI::App& app, + bool& opt_r, + std::string& mode, + std::string& path) +{ + app.name("chmod"); + app.description("Mode Interface"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeChmodHelp(); + })); + app.add_flag("-r", opt_r, "recursive"); + app.add_option("mode", mode, "mode (e.g. 755, 644, 700)")->required(); + app.add_option("path", path, "path")->required(); +} + +class ChmodCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "chmod"; + } + const char* + description() const override + { + return "Mode Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + bool opt_r = false; + std::string mode; + std::string path; + ConfigureChmodApp(app, opt_r, mode, path); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + XrdOucString in = "mgm.cmd=chmod"; + if (opt_r) + in += "&mgm.option=r"; + XrdOucString ap = abspath(path.c_str()); + in += "&mgm.path="; + in += ap; + in += "&mgm.chmod.mode="; + in += mode.c_str(); + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + bool opt_r = false; + std::string mode; + std::string path; + ConfigureChmodApp(app, opt_r, mode, path); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterChmodNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/chown-cmd-native.cc b/console/commands/native/chown-cmd-native.cc new file mode 100644 index 0000000000..cb3aeb8144 --- /dev/null +++ b/console/commands/native/chown-cmd-native.cc @@ -0,0 +1,139 @@ +// ---------------------------------------------------------------------- +// File: chown-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include + +namespace { +std::string MakeChownHelp() +{ + std::ostringstream oss; + oss << "Usage: chown [-r] [-h|--nodereference] [:] \n"; + oss << " chown [-r] : \n"; + oss << "'[eos] chown ..' provides the change owner interface of EOS.\n"; + oss << " is the file/directory to modify, has to be a user id " + "or user name. is optional and has to be a group id or group " + "name.\n"; + oss << "To modify only the group use : as identifier!\n"; + oss << "Remark: if you use the -r -h option and path points to a link the " + "owner of the link parent will also be updated!\n"; + oss << "Options:\n"; + oss << " -r recursive\n"; + oss << " -h, --nodereference don't follow symlinks\n"; + return oss.str(); +} + +void ConfigureChownApp(CLI::App& app, + bool& opt_r, + bool& opt_h, + std::string& owner, + std::string& path) +{ + app.name("chown"); + app.description("change owner interface of EOS"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeChownHelp(); + })); + app.add_flag("-r", opt_r, "recursive"); + app.add_flag("-h,--nodereference", opt_h, "don't follow symlinks"); + app.add_option("owner", owner, "owner[:group] or :group")->required(); + app.add_option("path", path, "path")->required(); +} + +class ChownCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "chown"; + } + const char* + description() const override + { + return "Chown Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str(), true); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str(), true)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + bool opt_r = false; + bool opt_h = false; + std::string owner; + std::string path; + ConfigureChownApp(app, opt_r, opt_h, owner, path); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::string opt; + if (opt_r) + opt += 'r'; + if (opt_h) + opt += 'h'; + + XrdOucString in = "mgm.cmd=chown"; + if (!opt.empty()) { + in += "&mgm.chown.option="; + in += opt.c_str(); + } + XrdOucString ap = abspath(path.c_str()); + in += "&mgm.path="; + in += ap; + in += "&mgm.chown.owner="; + in += owner.c_str(); + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + bool opt_r = false; + bool opt_h = false; + std::string owner; + std::string path; + ConfigureChownApp(app, opt_r, opt_h, owner, path); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterChownNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/clear-cmd-native.cc b/console/commands/native/clear-cmd-native.cc new file mode 100644 index 0000000000..e602d2e066 --- /dev/null +++ b/console/commands/native/clear-cmd-native.cc @@ -0,0 +1,92 @@ +// ---------------------------------------------------------------------- +// File: clear-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include +#include +#include +#include + +namespace { +std::string MakeClearHelp() +{ + return "Usage: clear\n" + "'[eos] clear' is equivalent to the interactive shell " + "command to clear the screen.\n"; +} + +void ConfigureClearApp(CLI::App& app) +{ + app.name("clear"); + app.description("Clear the terminal"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeClearHelp(); + })); +} + +class ClearCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "clear"; + } + const char* + description() const override + { + return "Clear the terminal"; + } + bool + requiresMgm(const std::string&) const override + { + return false; + } + int + run(const std::vector& args, CommandContext&) override + { + if (!args.empty()) { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + return 0; + } + } + + CLI::App app; + ConfigureClearApp(app); + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + return 0; + } + + int rc = system("clear"); + return rc; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeClearHelp().c_str()); + } +}; +} // namespace + +void +RegisterClearNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/config-proto-native.cc b/console/commands/native/config-proto-native.cc new file mode 100644 index 0000000000..2b2d9ccbb4 --- /dev/null +++ b/console/commands/native/config-proto-native.cc @@ -0,0 +1,205 @@ +// ---------------------------------------------------------------------- +// File: config-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include + +extern void com_config_help(); + +namespace { +std::string MakeConfigHelp() +{ + return "Usage: config changelog|dump|export|load|ls|reset|save [OPTIONS]\n\n" + "'[eos] config' provides the configuration interface to EOS.\n\n" + "Subcommands:\n" + " changelog [#lines] show last #lines from changelog (default 10)\n" + " dump [] dump configuration\n" + " export [-f] export config file to QuarkDB\n" + " load load config\n" + " ls [-b|--backup] list configurations\n" + " reset reset all configuration\n" + " save [-f] [-c|--comment \"\"] save config\n"; +} + +void ConfigureConfigApp(CLI::App& app) +{ + app.name("config"); + app.description("Configuration System"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeConfigHelp(); + })); +} + +// Ported from legacy com_proto_config.cc +class ConfigHelper : public ICmdHelper { +public: + ConfigHelper(const GlobalOptions& opts) : ICmdHelper(opts) {} + ~ConfigHelper() override = default; + bool + ParseCommand(const char* arg) override + { + eos::console::ConfigProto* config = mReq.mutable_config(); + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); + std::string token; + if (!tokenizer.NextToken(token)) { + return false; + } + if (token == "ls") { + eos::console::ConfigProto_LsProto* ls = config->mutable_ls(); + if (tokenizer.NextToken(token)) { + if (token == "--backup" || token == "-b") { + ls->set_showbackup(true); + } else { + return false; + } + } + } else if (token == "dump") { + eos::console::ConfigProto_DumpProto* dump = config->mutable_dump(); + if (tokenizer.NextToken(token)) { + dump->set_file(token); + } + } else if (token == "reset") { + if (tokenizer.NextToken(token)) { + return false; + } + config->set_reset(true); + } else if (token == "export") { + if (!tokenizer.NextToken(token)) { + return false; + } + eos::console::ConfigProto_ExportProto* exp = config->mutable_exp(); + if (token.find('-') != 0) { + exp->set_file(token); + if (tokenizer.NextToken(token)) { + if (token == "-f") { + exp->set_force(true); + } else { + return false; + } + } + } else { + return false; + } + } else if (token == "save") { + if (!tokenizer.NextToken(token)) { + return false; + } + eos::console::ConfigProto_SaveProto* save = config->mutable_save(); + if (token.find('-') != 0) { + save->set_file(token); + } else { + return false; + } + while (tokenizer.NextToken(token)) { + if (token == "-c" || token == "--comment") { + std::string sline = arg; + if (token == "-c") { + size_t pos = sline.find("-c"); + sline.replace(pos, std::string("-c").length(), "--comment"); + parse_comment(sline.c_str(), token); + } else { + parse_comment(sline.c_str(), token); + } + mReq.set_comment(token); + tokenizer.NextToken(token); + } else if (token == "-f") { + save->set_force(true); + } else { + return false; + } + } + } else if (token == "load") { + if (!tokenizer.NextToken(token)) { + return false; + } + eos::console::ConfigProto_LoadProto* load = config->mutable_load(); + load->set_file(token); + } else if (token == "changelog") { + eos::console::ConfigProto_ChangelogProto* changelog = + config->mutable_changelog(); + if (tokenizer.NextToken(token)) { + if (token.find('-') == 0) { + token.erase(0); + } + try { + changelog->set_lines(std::stoi(token)); + } catch (...) { + std::cerr << "error: argument needs to be numeric" << std::endl; + return false; + } + } else { + changelog->set_lines(10); + } + } else { + return false; + } + return true; + } +}; + +class ConfigProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "config"; + } + const char* + description() const override + { + return "Configuration System"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + ConfigHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureConfigApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterConfigProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/convert-proto-native.cc b/console/commands/native/convert-proto-native.cc new file mode 100644 index 0000000000..35e394d9c3 --- /dev/null +++ b/console/commands/native/convert-proto-native.cc @@ -0,0 +1,233 @@ +// ---------------------------------------------------------------------- +// File: convert-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/LayoutId.hh" +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include + +namespace { +std::string MakeConvertHelp() +{ + return "Usage: convert config|file|list|rule|clear [OPTIONS]\n\n" + " config list|set [=]\n" + " list: list converter configuration parameters and status\n" + " set : set converter configuration parameters\n\n" + " list\n" + " list conversion jobs\n\n" + " clear\n" + " clear list of jobs stored in the backend\n\n" + " file \n" + " schedule a file conversion\n" + " = fid|fxid|path\n" + " = [space] [placement] [checksum]\n\n" + " rule \n" + " apply a conversion rule on the given directory\n" + " = cid|cxid|path\n" + " = [space] [placement] [checksum]\n"; +} + +void ConfigureConvertApp(CLI::App& app) +{ + app.name("convert"); + app.description("Convert Interface"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeConvertHelp(); + })); +} + +class ConvertHelper : public ICmdHelper { +public: + ConvertHelper(const GlobalOptions& opts) : ICmdHelper(opts) {} + ~ConvertHelper() override = default; + bool + ParseCommand(const char* arg) override + { + using eos::console::ConvertProto_ConfigProto; + eos::console::ConvertProto* convert = mReq.mutable_convert(); + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); + std::string token; + if (!tokenizer.NextToken(token)) + return false; + if (token == "config") { + ConvertProto_ConfigProto* config = convert->mutable_config(); + if (!tokenizer.NextToken(token)) + return false; + if (token == "list") { + config->set_op(ConvertProto_ConfigProto::LIST); + } else if (token == "set") { + if (!tokenizer.NextToken(token)) { + return false; + } + size_t pos = token.find('='); + if (pos == std::string::npos || pos == token.length() - 1) + return false; + config->set_op(ConvertProto_ConfigProto::SET); + config->set_key(token.substr(0, pos)); + config->set_value(token.substr(pos + 1)); + } else { + return false; + } + } else if (token == "file") { + eos::console::ConvertProto_FileProto* file = convert->mutable_file(); + if (!tokenizer.NextToken(token)) + return false; + file->set_allocated_identifier(ParseIdentifier(token)); + eos::console::ConvertProto_ConversionProto* conversion = + ParseConversion(tokenizer); + if (conversion == nullptr) + return false; + file->set_allocated_conversion(conversion); + } else if (token == "rule") { + if (!tokenizer.NextToken(token)) + return false; + eos::console::ConvertProto_RuleProto* rule = convert->mutable_rule(); + rule->set_allocated_identifier(ParseIdentifier(token)); + eos::console::ConvertProto_ConversionProto* conversion = + ParseConversion(tokenizer); + if (conversion == nullptr) + return false; + rule->set_allocated_conversion(conversion); + } else if (token == "list") { + convert->mutable_list(); + } else if (token == "clear") { + convert->mutable_clear(); + } else { + return false; + } + return true; + } + +private: + eos::console::ConvertProto_IdentifierProto* + ParseIdentifier(std::string spath) + { + XrdOucString path = spath.c_str(); + auto* identifier = new eos::console::ConvertProto_IdentifierProto{}; + unsigned long long id = 0ull; + if (Path2FileDenominator(path, id)) + identifier->set_fileid(id); + else if (Path2ContainerDenominator(path, id)) + identifier->set_containerid(id); + else + identifier->set_path(abspath(path.c_str())); + return identifier; + } + + eos::console::ConvertProto_ConversionProto* + ParseConversion(eos::common::StringTokenizer& tokenizer) + { + std::string token, layout, space, placement, checksum; + int replica = 0; + size_t pos; + bool ok; + auto validLayout = [](const std::string& l) { + return eos::common::LayoutId::GetLayoutFromString(l) != -1; + }; + auto validPlacement = [](const std::string& p) { + return (p == "scattered" || p == "hybrid" || p == "gathered"); + }; + auto validChecksum = [](const std::string& c) { + using eos::common::LayoutId; + auto xs_id = LayoutId::GetChecksumFromString(c); + return ((xs_id > -1) && (xs_id != LayoutId::eChecksum::kNone)); + }; + if (!tokenizer.NextToken(token)) + return nullptr; + if ((pos = token.find(":")) == std::string::npos) + return nullptr; + layout = token.substr(0, pos); + try { + replica = std::stol(token.substr(pos + 1)); + } catch (...) { + return nullptr; + } + if (!validLayout(layout)) + return nullptr; + if (replica < 1 || replica > 32) + return nullptr; + while (tokenizer.NextToken(token)) { + if ((ok = validChecksum(token))) + checksum = std::move(token); + else if ((ok = validPlacement(token))) + placement = std::move(token); + else if ((ok = space.empty())) + space = std::move(token); + if (!ok) + return nullptr; + } + auto* conversion = new eos::console::ConvertProto_ConversionProto{}; + conversion->set_layout(layout); + conversion->set_replica(replica); + conversion->set_space(space); + conversion->set_placement(placement); + conversion->set_checksum(checksum); + return conversion; + } +}; + +class ConvertProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "convert"; + } + const char* + description() const override + { + return "Convert Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + ConvertHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureConvertApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterConvertProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/cp-cmd-native.cc b/console/commands/native/cp-cmd-native.cc new file mode 100644 index 0000000000..45c9a46af7 --- /dev/null +++ b/console/commands/native/cp-cmd-native.cc @@ -0,0 +1,1778 @@ +// ---------------------------------------------------------------------- +// File: cp-cmd-native.cc +// Author: Andreas-Joachim Peters - CERN +// ---------------------------------------------------------------------- + +/************************************************************************ + * EOS - the CERN Disk Storage System * + * Copyright (C) 2011 CERN/Switzerland * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see .* + ************************************************************************/ + +/*----------------------------------------------------------------------------*/ +#include +#include +#include +#include +#include +#include +#include +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include "common/Path.hh" +#include "common/StringConversion.hh" +#include +#include +#include +#include +#include +/*----------------------------------------------------------------------------*/ + +/** Parsed options for cp command */ +struct CpOptions { + std::string rate; + std::string streams = "0"; + std::string atomic; + std::vector sources; + std::string target; + bool recursive = false; + bool summary = false; + bool noprogress = false; + bool append = false; + bool makeparent = false; + bool debug = false; + int debug_level = 0; + bool checksums = false; + bool silent = false; + bool nooverwrite = false; + bool preserve = false; + unsigned long depth = 0; +}; + +namespace { +std::string MakeCpHelp() +{ + return R"(Usage: cp [OPTIONS] ... + +'[eos] cp ..' provides copy functionality to EOS. +| can be root:///, a local path /tmp/../ or an eos path +/eos/ in the connected instance. + +Options: + --atomic run an atomic upload (files visible only when complete) + --rate= limit the cp rate + --streams= use <#> parallel streams + --depth= depth for recursive copy + --checksum output the checksums + -a append to the target, don't truncate + -p create destination directory + -n hide progress bar + -S print summary + -d, --debug[=1|2|3] enable debug information + -s, --silent no output outside error messages + -k, --no-overwrite disable overwriting of files + -P, --preserve preserve file creation and modification time + -r, -R, --recursive copy source location recursively + +Remark: + Add '/' at the end of source or target paths for directories. Use '-r' with + source and target directories terminated with '/' for hierarchy copy. + +Examples: + eos cp /var/data/myfile /eos/foo/user/data/ + eos cp /var/data/ /eos/foo/user/data/ + eos cp -r /var/data/ /eos/foo/user/data/ + eos cp -r --checksum --silent /var/data/ /eos/foo/user/data/ + +S3: + URLs: as3://// or as3:/ with S3_HOSTNAME set. + Credentials: S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_HOSTNAME + or s3.id=, s3.key= in URL. +)"; +} + +void ConfigureCpApp(CLI::App& app, CpOptions& opts) +{ + app.name("cp"); + app.description("Copy files to/from EOS"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeCpHelp(); + })); + app.add_option("--rate", opts.rate, "limit cp rate"); + app.add_option("--streams", opts.streams, "parallel streams")->default_val("0"); + app.add_flag("--atomic", [&](size_t) { opts.atomic = "&eos.atomic=1"; }, + "atomic upload"); + app.add_option("--depth", opts.depth, "recursive copy depth"); + app.add_flag("--checksum", opts.checksums, "output checksums"); + app.add_flag("-a,--append", opts.append, "append, don't truncate"); + app.add_flag("-p", opts.makeparent, "create destination directory"); + app.add_flag("-n", opts.noprogress, "hide progress bar"); + app.add_flag("-S", opts.summary, "print summary"); + app.add_flag("-d,--debug", [&](size_t c) { + if (c) + opts.debug_level = 1; + }, "enable debug"); + app.add_flag("-s,--silent", opts.silent, "silent mode"); + app.add_flag("-k,--no-overwrite", opts.nooverwrite, "no overwrite"); + app.add_flag("-P,--preserve", opts.preserve, "preserve mtime"); + app.add_flag("-r,-R,--recursive", opts.recursive, "recursive copy"); + app.add_option("path", opts.sources, "source paths and destination") + ->required() + ->expected(2, -1); +} + +/** Parse cp args with CLI11. Returns true on success, false on help/error. */ +bool ParseCpArgs(const std::vector& args, CpOptions& opts) +{ + if (args.empty()) + return false; + + for (const auto& a : args) { + if (a == "--help" || a == "-h") { + std::cerr << MakeCpHelp(); + return false; + } + } + + CLI::App app; + ConfigureCpApp(app, opts); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError& e) { + std::cerr << MakeCpHelp(); + return false; + } + + if (opts.sources.size() < 2) { + std::cerr << "error: at least one source and one destination required\n"; + return false; + } + + opts.target = opts.sources.back(); + opts.sources.pop_back(); + opts.debug = (opts.debug_level > 0); + if (opts.recursive) + opts.makeparent = true; + if (opts.silent || !hasterminal) + opts.noprogress = true; + + return true; +} + +/** Tokenize char* into vector of strings for CLI11 parsing */ +std::vector TokenizeCpArgs(const char* argin) +{ + std::vector args; + eos::common::StringTokenizer tokenizer(argin); + tokenizer.GetLine(); + XrdOucString tok; + while ((tok = tokenizer.GetToken()).length()) { + if ((!tok.beginswith("/eos/")) && (!tok.beginswith("root:/"))) + eos::common::StringConversion::UnsealXrdPath(tok); + args.push_back(tok.c_str()); + } + return args; +} +} // namespace + +static int com_cp_usage() +{ + std::cerr << MakeCpHelp(); + return EINVAL; +} + +/* Helper types */ +enum Protocol { + HTTP, HTTPS, GSIFTP, + S3, AS3, XROOT, + EOS, LOCAL, UNKNOWN +}; + +struct File_t { + XrdOucString name; + XrdOucString opaque; + Protocol protocol; + timespec atime; + timespec mtime; + unsigned long long size; + + File_t() : name(""), opaque(""), protocol(Protocol::UNKNOWN), size(0) { } +}; + +/* Helper functions */ +int run_eos_command(const char* cmdline, std::vector& result); +int run_command(const char* cmdline, std::vector& result); +const char* absolute_path(const char* path); +bool is_dir(const char* path, Protocol protocol, struct stat* buf = NULL); +XrdOucString process_symlink(XrdOucString path); +const char* setup_s3_environment(XrdOucString path, XrdOucString opaque); +std::string eos_roles_opaque(); +int do_stat(const char* path, Protocol protocol, struct stat& buf); +int check_protocol_tool(const char* path); +Protocol get_protocol(XrdOucString path); +const char* protocol_to_string(Protocol protocol); + +/** Core copy implementation */ +static int cp_impl(const CpOptions& opts) +{ + XrdOucString rate = opts.rate.c_str(); + XrdOucString streams = opts.streams.c_str(); + XrdOucString atomic = opts.atomic.c_str(); + std::vector source_find_list; + for (const auto& s : opts.sources) + source_find_list.emplace_back(s.c_str()); + std::vector source_basepath_list; + std::vector source_list; + File_t target; + target.name = opts.target.c_str(); + bool target_is_stdout; + bool target_is_dir = false; + bool recursive = opts.recursive; + bool summary = opts.summary; + bool noprogress = opts.noprogress; + bool append = opts.append; + bool makeparent = opts.makeparent; + bool debug = opts.debug; + int debug_level = opts.debug_level; + bool checksums = opts.checksums; + bool silent = opts.silent; + bool nooverwrite = opts.nooverwrite; + bool preserve = opts.preserve; + unsigned long long copysize = 0; + unsigned long long copiedsize = 0; + unsigned long depth = opts.depth; + struct timeval start_time, end_time; + struct timezone tz; + int files_copied = 0; + int retc = 0; + + if (!target.name.length()) { + std::cerr << "warning: no target specified. Please view 'eos cp --help'." + << std::endl; + global_retc = 0; + return 0; + } + + // -------------------------------------------------------------------------- + // Expand source list into final list to copy. + // This means interpreting the '*' character in file names + // and traversing directories for the recursive flag. + // Every source path also has an associated base path, + // which will get appended to the target. + // -------------------------------------------------------------------------- + + for (size_t i = 0; i < source_find_list.size(); i++) { + std::vector files; + XrdOucString source = source_find_list[i]; + XrdOucString source_opaque; + XrdOucString basepath = ""; + Protocol protocol; + std::string sprotocol = ""; + int opos = source.find("?"); + bool wildcard = false; + files.clear(); + + // Extract opaque info + if (opos != STR_NPOS) { + source_opaque = source; + source_opaque.erase(0, opos + 1); + source.erase(opos); + } + + // Identify protocol + protocol = get_protocol(source.c_str()); + + if (protocol == Protocol::UNKNOWN) { + std::cerr << "warning: " << source.c_str() + << " -- protocol not recognized. Skipping path.." << std::endl; + continue; + } + + // Convert local to absolute path + const char* abs_path = absolute_path(source.c_str()); + source = abs_path; + free((char*)abs_path); + + // Check if source is a directory + if (!source.endswith("/") && is_dir(source.c_str(), protocol, NULL)) { + source.append("/"); + } + + // Extract file name and parent path + const char* filepath = source.c_str(); + + // URLs need different processing in order to extract the path + if ((protocol != Protocol::EOS) && (protocol != Protocol::LOCAL)) { + XrdOucString sprot, hostport; + filepath = eos::common::StringConversion::ParseUrl(source.c_str(), + sprot, hostport); + + if (!filepath) { + std::cerr << "error: cannot process file=" << source.c_str() + << " [protocol=" << protocol_to_string(protocol) << "]" + << std::endl; + continue; + } + } + + eos::common::Path cPath(filepath); + basepath = cPath.GetParentPath(); + + if ((source.find("*") != STR_NPOS) || (source.endswith("/"))) { + std::string cmdtext; + + if ((protocol != Protocol::EOS) && (protocol != Protocol::LOCAL)) { + std::cerr << "error: " << source.c_str() + << " -- path expansion not implemented for " + << protocol_to_string(protocol) << " protocol. Skipping path.." + << std::endl; + continue; + } + + // Get all paths matching wildcard + if (source.find("*") != STR_NPOS) { + // Will use 'ls -lF' combined with grep to identify matches + // ls -l[F|p] | awk 'NF == 9 {print $9}' [ | egrep "" ] + // Note: eos::common::Path removes trailing '/'! + XrdOucString basename = cPath.GetName(); + + if (source.endswith("/")) { + basename.append("/"); + } + + // Wildcards are supported only in the basename + if (basename.find("*") == STR_NPOS) { + std::cerr << "warning: " << source.c_str() + << " -- wildcards not supported outside basename. " + "Skipping path.." << std::endl; + continue; + } + + XrdOucString match = basename.c_str(); + wildcard = true; + + if (!match.beginswith("*")) { + match.insert("^", 0); + } + + if (!match.endswith("*")) { + match.append("$"); + } + + match.replace("*", ".*"); + // Construct command text + cmdtext = "ls -l"; + cmdtext += (protocol == Protocol::EOS) ? "F " : "p "; + cmdtext += basepath.c_str(); + cmdtext += + " | awk '{out=$9; for (i=10; i<=NF; i++) {out=out\" \"$i}; print out}' | egrep \""; + cmdtext += match.c_str(); + cmdtext += "\""; + } else if (source.endswith("/")) { + // Get all files within directory + + // Will use 'find' to identify files + // local file: find [-maxdepth ] -follow -type f + // eos file: find -f [--maxdepth ] + if (!recursive) { + std::cerr << "warning: omitting directory " << source.c_str() + << std::endl; + continue; + } + + // Enclose source path in quotes, as the path may contain whitespace + std::stringstream ss; + ss.clear(); + ss << std::quoted(source.c_str()); + source = ss.str().c_str(); + // Capture only last directory + // This will end up appended to the target + std::string smaxdepth = " "; + + if (depth != 0) { + smaxdepth = " -maxdepth "; + smaxdepth += std::to_string(depth); + smaxdepth += " "; + + if (protocol == Protocol::EOS) { + smaxdepth.insert(1, "-"); + } + } + + cmdtext = "find "; + + if (protocol == Protocol::EOS) { + cmdtext += "-f"; + cmdtext += smaxdepth.c_str(); + cmdtext += source.c_str(); + } else { + cmdtext += source.c_str(); + cmdtext += smaxdepth.c_str(); + cmdtext += "-follow -type f"; + } + } + + cmdtext += " 2> /dev/null"; + + if (debug) { + std::cerr << "[eos-cp] running: " << cmdtext.c_str() << std::endl; + } + + int rc = (protocol == Protocol::EOS) ? + run_eos_command(cmdtext.c_str(), files) : + run_command(cmdtext.c_str(), files); + + if (rc && !files.size()) { + std::cerr << "warning: could not expand source: " << source.c_str() + << std::endl; + global_retc = rc; + return -1; + } + } else { + files.emplace_back(source.c_str()); + } + + for (auto& file : files) { + // Check if path expansion discovered a symlink + if (file.find(" -> ") != STR_NPOS) { + file = process_symlink(file.c_str()); + } + + if (wildcard) { + file.insert(basepath.c_str(), 0); + source_find_list.emplace_back(file.c_str()); + continue; + } + + if (debug) { + std::cerr << "[eos-cp] Copy list: " << file.c_str() << std::endl; + } + + File_t source_file; + source_file.name = file.c_str(); + source_file.opaque = source_opaque.c_str(); + if (getenv("EOSAUTHZ")) { + source_file.opaque += "&authz="; + source_file.opaque += getenv("EOSAUTHZ"); + } + source_file.protocol = protocol; + source_list.emplace_back(source_file); + source_basepath_list.emplace_back(basepath.c_str()); + } + } + + // Check if there is any file in the list + if (source_list.empty()) { + std::cerr << "warning: found zero files to copy!" << std::endl; + global_retc = 0; + return 0; + } + + // -------------------------------------------------------------------------- + // Process target path + // -------------------------------------------------------------------------- + bool target_exists; + struct stat target_stat; + target.protocol = get_protocol(target.name.c_str()); + + // Make sure executable to reach target exists + if (check_protocol_tool(target.name.c_str())) { + return -1; + } + + // Handle opaque information for target + if (target.protocol != Protocol::LOCAL) { + int qpos = target.name.find("?"); + + if (qpos != STR_NPOS) { + target.opaque = target.name.c_str(); + target.opaque.keep(qpos + 1); + target.name.erase(qpos); + } + + // Seal the target name + if (target.protocol == Protocol::EOS) { + eos::common::StringConversion::SealXrdPath(target.name); + } + } + + // Detect whether target is stdout + const char* abs_path = absolute_path(target.name.c_str()); + target.name = abs_path; + free((char*)abs_path); + target_is_stdout = (target.name == "-"); + + if (!target_is_stdout) { + // Detect whether target is a directory + int stat_rc = do_stat(target.name.c_str(), target.protocol, target_stat); + target_exists = (stat_rc == 0); + // Only pass stat buffer when stat succeeded - otherwise is_dir would use + // undefined buffer content and could spuriously return true + target_is_dir = is_dir(target.name.c_str(), target.protocol, + target_exists ? &target_stat : nullptr); + + // If multiple source files target must be a directory + if (source_list.size() > 1) { + // Target doesn't exist, mark it as directory + if (!target_exists) { + target_is_dir = true; + } + + // Target is not a directory + if (!target_is_dir) { + std::cerr << "error: target must be a directory" << std::endl; + global_retc = EINVAL; + return -1; + } + } + + // Target doesn't exist but name suggests should be a directory + if (!target_exists && target.name.endswith("/")) { + target_is_dir = true; + } + + // If target is a directory then the name should also reflect this + if (target_is_dir && !target.name.endswith("/")) { + target.name.append("/"); + } + + // Check rights to create target directory + if (target_is_dir && !target_exists) { + if (!makeparent) { + std::cerr << "error: target must be created. Please try with " + "create flag '-p' or see 'eos cp --help' for more info." + << std::endl; + global_retc = EINVAL; + return -1; + } + } + + // Create target directory tree for EOS or local path + if (makeparent) { + if ((target.protocol == Protocol::EOS) || + (target.protocol == Protocol::LOCAL)) { + XrdOucString mktarget; + + if (target.name.endswith("/")) { + mktarget = target.name.c_str(); + } else { + eos::common::Path cTarget(target.name.c_str()); + mktarget = cTarget.GetParentPath(); + } + + std::string cmdtext = "mkdir -p "; + + if (target.protocol == Protocol::LOCAL) { + cmdtext += "--mode 755 "; + } + + cmdtext += mktarget.c_str(); + std::vector tmp; + int rc = (target.protocol == Protocol::EOS) ? + run_eos_command(cmdtext.c_str(), tmp) : + run_command(cmdtext.c_str(), tmp); + + if (rc) { + std::cerr << "error: failed to create target directory : " + << mktarget.c_str() << std::endl; + global_retc = rc; + return -1; + } + } + } + } else { + // Disable all output for stdout target + silent = true; + noprogress = true; + } + + // Set up environment for S3 target + if ((target.protocol == Protocol::AS3) || + (target.protocol == Protocol::S3)) { + const char* url = setup_s3_environment(target.name, target.opaque); + + if (url == NULL) { + return -1; + } + + target.name = url; + } + + // Expand '/eos/' shortcut for EOS protocol + if ((target.protocol == Protocol::EOS) && + (target.name.beginswith("/eos/"))) { + if (!serveruri.endswith("/")) { + target.name.insert("/", 0); + } + + target.name.insert(serveruri.c_str(), 0); + if (getenv("EOSAUTHZ")) { + target.opaque+= "&authz="; + target.opaque+= getenv("EOSAUTHZ"); + } + } + + if (debug) { + std::cerr << "[eos-cp] # of source files: " << source_list.size() + << std::endl; + std::cerr << "[eos-cp] Setting target " << target.name.c_str() + << " [protocol=" << protocol_to_string(target.protocol) << "]" + << std::endl; + } + + // -------------------------------------------------------------------------- + // Compute size for each source path + // -------------------------------------------------------------------------- + // As needed, check whether tools to access these protocols can be found + bool s3_tool = false; + bool http_tool = false; + bool gsiftp_tool = false; + + for (auto& source : source_list) { + bool statok = false; + struct stat buf; + source.atime.tv_nsec = source.mtime.tv_nsec = 0; + + switch (source.protocol) { + // ------------------------------------------ + // EOS, XRoot or local file + // ------------------------------------------ + case Protocol::EOS: + case Protocol::XROOT: + case Protocol::LOCAL: + if (!do_stat(source.name.c_str(), source.protocol, buf)) { + // For symbolic links, EOS stat returns the size of the link. + // Ignore the size attribute in this case + if (source.protocol != Protocol::LOCAL && !S_ISREG(buf.st_mode)) { + source.size = 0; + + if (debug || !silent) { + std::cerr << "warning: disable size check for path=" + << source.name.c_str() << " [EOS symbolic link]" + << std::endl; + } + } else { + copysize += buf.st_size; + source.size = (unsigned long long) buf.st_size; + } + + // Store the a/m-time + source.atime.tv_sec = buf.st_atime; + source.mtime.tv_sec = buf.st_mtime; + statok = true; + } + + break; + + // ------------------------------------------ + // S3 file + // ------------------------------------------ + case Protocol::AS3: + case Protocol::S3: { + if (!s3_tool) { + if (check_protocol_tool(source.name.c_str())) { + return -1; + } + + s3_tool = true; + } + + const char* url = setup_s3_environment(source.name, source.opaque); + + if (url == NULL) { + return -1; + } + + XrdOucString s3env = "env S3_ACCESS_KEY_ID="; + s3env += getenv("S3_ACCESS_KEY_ID"); + s3env += " S3_HOSTNAME="; + s3env += getenv("S3_HOSTNAME"); + s3env += " S3_SECRET_ACCESS_KEY="; + s3env += getenv("S3_SECRET_ACCESS_KEY"); + // Execute 's3' command to retrieve size + XrdOucString cmdtext = "bash -c \""; + cmdtext += s3env; + cmdtext += " s3 head "; + cmdtext += url; + cmdtext += " | grep Content-Length | awk '{print \\$2}' 2> /dev/null\""; + + if (debug) { + std::cerr << "[eos-cp] running " << cmdtext.c_str() << std::endl; + } + + long long size = eos::common::StringConversion::LongLongFromShellCmd( + cmdtext.c_str()); + + if ((!size) || (size == LLONG_MAX)) { + std::cerr << "error: path=" << source.name.c_str() + << " cannot obtain size of S3 source file or file size is 0!" + << std::endl; + global_retc = EIO; + return -1; + } + + copysize += size; + source.size = (unsigned long long) size; + source.atime.tv_sec = source.mtime.tv_sec = 0; + statok = true; + break; + } + + // ------------------------------------------ + // HTTP(S) & GSIFTP file + // ------------------------------------------ + case Protocol::GSIFTP: + case Protocol::HTTP: + case Protocol::HTTPS: + if ((source.protocol == Protocol::HTTP || + source.protocol == Protocol::HTTPS) && (!http_tool)) { + if (check_protocol_tool(source.name.c_str())) { + return -1; + } + + http_tool = true; + } else if ((source.protocol == Protocol::GSIFTP) && (!gsiftp_tool)) { + if (check_protocol_tool(source.name.c_str())) { + return -1; + } + + gsiftp_tool = true; + } + + source.size = 0; + source.atime.tv_sec = source.mtime.tv_sec = 0; + + if (debug || !silent) { + std::cerr << "warning: disabling size check for path=" + << source.name.c_str() << " [protocol=" + << protocol_to_string(source.protocol) << "]" << std::endl; + } + + statok = true; + break; + + default: + break; + } + + if (!statok) { + std::cerr << "error: cannot get file size of path=" << source.name.c_str() + << " [protocol=" << protocol_to_string(source.protocol) << "]" + << std::endl; + global_retc = EINVAL; + return -1; + } + + if (debug) { + std::cerr << "[eos-cp] path=" << source.name.c_str() << " size=" + << source.size << " [protocol=" + << protocol_to_string(source.protocol) << "]" << std::endl; + } + } + + if (debug || (!silent && source_list.size() > 1)) { + XrdOucString ssize; + std::cerr << "[eos-cp] going to copy " << source_list.size() << " files and " + << eos::common::StringConversion::GetReadableSizeString(ssize, + copysize, "B") << std::endl; + } + + // Mark start timestamp + gettimeofday(&start_time, &tz); + // -------------------------------------------------------------------------- + // Create 'eoscp' command for each source path + // and effectively perform the copy operation + // -------------------------------------------------------------------------- + int file_idx = -1; + retc = 0; + + for (auto& source : source_list) { + XrdOucString dest = target.name.c_str(); + // Processed target path + original target opaque info + XrdOucString target_path = ""; + // Temporary file upload flag + bool temporary_file = false; + file_idx++; + + //------------------------------------ + // Process destination path + //------------------------------------ + + // Append source suffix to destination + // The source suffix: = + if (target_is_dir) { + XrdOucString source_suffix = source.name.c_str(); + int pos = source_suffix.find(source_basepath_list[file_idx].c_str()); + + if (pos == STR_NPOS) { + std::cerr << "error: could not identify source suffix for path=" + << source.name.c_str() << std::endl; + global_retc = EINVAL; + return -1; + } + + pos += source_basepath_list[file_idx].length(); + source_suffix.keep(pos); + dest += source_suffix.c_str(); + } + + // Check that source and destination are different + if (!strcmp(source.name.c_str(), dest.c_str())) { + std::cerr << "warning: source and target are the same path=" + << source.name.c_str() << ". Skipping path.." << std::endl; + continue; + } + + // Add opaque info to destination + if (target.opaque.length()) { + dest += "?"; + dest += target.opaque.c_str(); + } + + target_path = dest.c_str(); + + // Continue processing for non STDOUT targets + if (!target_is_stdout) { + // Check if destination exists + if (nooverwrite) { + if ((target.protocol == Protocol::LOCAL) || + (target.protocol == Protocol::EOS)) { + struct stat tmp; + + if (!do_stat(dest.c_str(), target.protocol, tmp)) { + std::cerr << "warning: target=" << dest.c_str() + << " exists, but --no-overwrite flag specified" + << std::endl; + retc |= EEXIST; + continue; + } + } + } + + // Handle EOS specific opaque info + if ((target.protocol == Protocol::EOS) || + (target.protocol == Protocol::XROOT)) { + char opaque[1024]; + std::string roles = eos_roles_opaque(); + snprintf(opaque, sizeof(opaque) - 1, + "%ceos.targetsize=%llu&eos.bookingsize=%llu&eos.app=%s%s%s%s", + (target.opaque.length()) ? '&' : '?', + source.size, source.size, getenv("EOSAPP") ? getenv("EOSAPP") : "eoscp", + atomic.c_str(), + roles.size() ? "&" : "", + roles.size() ? roles.c_str() : ""); + dest.append(opaque); + } + + // Protocols for EOS, XRoot and local targets are supported directly + // S3 targets will be uploaded via STDIN & STDOUT pipes + // Remaining protocols will be copied to a temporary file + if ((target.protocol == Protocol::HTTP) || + (target.protocol == Protocol::HTTPS) || + (target.protocol == Protocol::GSIFTP)) { + char tmp_name[] = "/tmp/com_cp.XXXXXX"; + int tmp_fd = mkstemp(tmp_name); + + if (tmp_fd == -1) { + std::cerr << "error: failed to create temporary file while preparing " + << "copy for path=" << dest.c_str() << " [protocol=" + << protocol_to_string(target.protocol) << "]" << std::endl; + global_retc = errno; + return -1; + } + + close(tmp_fd); + temporary_file = true; + dest = tmp_name; + } + } + + //------------------------------------ + // Process source path + //------------------------------------ + + // Expand '/eos/' shortcut for EOS protocol + if ((source.protocol == Protocol::EOS) && + (source.name.beginswith("/eos/"))) { + if (!serveruri.endswith("/")) { + source.name.insert("/", 0); + } + + source.name.insert(serveruri.c_str(), 0); + } + + // Add opaque info to source + if (source.opaque.length()) { + source.name += "?"; + source.name += source.opaque.c_str(); + } + + if (debug) { + std::cerr << "\n[eos-cp] copying " << source.name.c_str() << " to " + << target_path.c_str() << std::endl; + } + + //------------------------------------ + // Prepare STDIN and STDOUT pipes + //------------------------------------ + XrdOucString transfersize = + ""; // used for STDIN pipes to specify the target size to eoscp + XrdOucString cmdtext = ""; + bool rstdin = false; + bool rstdout = false; + + if ((source.protocol == Protocol::EOS) || + (source.protocol == Protocol::XROOT)) { + std::string roles = eos_roles_opaque(); + source.name += (source.opaque.length()) ? "&" : "?"; + source.name += "eos.app="; + source.name += getenv("EOSAPP") ? getenv("EOSAPP") : "eoscp"; + source.name += roles.size() ? "&" : ""; + source.name += roles.size() ? roles.c_str() : ""; + } else if ((source.protocol != Protocol::LOCAL) && + (source.protocol != Protocol::UNKNOWN)) { + bool old_noprogress = noprogress; + noprogress = true; + XrdOucString safesource = source.name.c_str(); + + while (safesource.replace("'", "\\'")) {} + + safesource.replace("as3:", "", 0, 3); + XrdOucString tool = ""; + + if (source.protocol == Protocol::HTTP) { + tool = "curl "; + } + + if (source.protocol == Protocol::HTTPS) { + tool = "curl -k "; + } + + if (source.protocol == Protocol::GSIFTP) { + tool = "globus-url-copy "; + } + + if ((source.protocol == Protocol::AS3) || + (source.protocol == Protocol::S3)) { + tool = "s3 get "; + noprogress = old_noprogress; + } + + cmdtext += tool; + cmdtext += "$'"; + cmdtext += safesource; + cmdtext += "'"; + + if (source.protocol == Protocol::GSIFTP) { + cmdtext += " -"; + } + + cmdtext += " | "; + rstdin = true; + } + + if ((source.protocol == Protocol::AS3) || + (source.protocol == Protocol::S3) || + (target.protocol == Protocol::AS3) || + (target.protocol == Protocol::S3)) { + char ts[1024]; + snprintf(ts, sizeof(ts) - 1, "%llu ", source.size); + transfersize = ts; + } + + if ((target.protocol == Protocol::AS3) || + (target.protocol == Protocol::S3)) { + rstdout = true; + } + + //------------------------------------ + // Prepare eoscp transaction name + //------------------------------------ + XrdOucString safename = source.name.c_str(); + int qpos = safename.rfind("?"); + + if (qpos != STR_NPOS) { + safename.erase(qpos); + } + + if (source.protocol != Protocol::LOCAL) { + XrdOucString sprot, hostport; + const char* url = eos::common::StringConversion::ParseUrl(safename.c_str(), + sprot, hostport); + + if (url) { + std::string surl = url; + safename = surl.c_str(); + } + } + + safename = eos::common::Path(safename.c_str()).GetName();; + eos::common::StringConversion::SealXrdPath(safename); + safename.replace("'", "\\'"); + //------------------------------------ + // Construct 'eoscp' command + //------------------------------------ + cmdtext += "eoscp "; + + if (append) { + cmdtext += "-a "; + } + + if (debug_level) { + cmdtext += (debug_level == 1) ? "-v " : "-d "; + } + + if (!summary) { + cmdtext += "-s "; + } + + if (makeparent) { + cmdtext += "-p "; + } + + if (noprogress) { + cmdtext += "-n "; + } + + if (nooverwrite) { + cmdtext += "-x "; + } + + if (transfersize.length()) { + cmdtext += "-T "; + cmdtext += transfersize; + cmdtext += " "; + } + + if (rate.length()) { + cmdtext += "-t "; + cmdtext += rate.c_str(); + cmdtext += " "; + } + + cmdtext += "-N $'"; + cmdtext += safename.c_str(); + cmdtext += "' "; + + if (rstdin) { + cmdtext += "- "; + } else { + XrdOucString safesource = source.name.c_str(); + safesource.replace("'", "\\'"); + cmdtext += "$'"; + cmdtext += safesource; + cmdtext += "' "; + } + + if (rstdout) { + cmdtext += "-"; + } else { + XrdOucString safedest = dest.c_str(); + safedest.replace("'", "\\'"); + cmdtext += "$'"; + cmdtext += safedest; + cmdtext += "'"; + } + + if ((target.protocol == Protocol::AS3) || + (target.protocol == Protocol::S3)) { + // s3 can upload via STDIN + XrdOucString s3dest = dest.c_str(); + s3dest.replace("as3:", "", 0, 3); + cmdtext += " | s3 put "; + cmdtext += s3dest.c_str(); + cmdtext += " contentLength="; + cmdtext += transfersize.c_str(); + cmdtext += " > /dev/null"; + } + + if (debug) { + std::cerr << "[eos-cp] running: " << cmdtext.c_str() << std::endl; + } + + int lrc = system(cmdtext.c_str()); + + // Check if we got a CONTROL-C + if (lrc == EINTR) { + std::cerr << "" << std::endl; + break; + } + + if (WEXITSTATUS(lrc)) { + std::cerr << "error: failed copying path=" << target_path.c_str() + << std::endl; + retc |= lrc; + continue; + } + + //------------------------------------ + // Check target size + //------------------------------------ + + if (((target.protocol == Protocol::EOS) || + (target.protocol == Protocol::XROOT) || + (target.protocol == Protocol::LOCAL)) && (!target_is_stdout)) { + struct stat buf; + + if (!do_stat(target_path.c_str(), target.protocol, buf)) { + if ((!source.size) || + (buf.st_size == (off_t)(append ? target_stat.st_size + source.size : + (off_t) source.size) + ) + ) { + // Preserve creation and modification timestamps + if ((preserve) && (source.atime.tv_sec > 0) && (source.mtime.tv_sec > 0)) { + bool updateok; + + if (target.protocol == Protocol::LOCAL) { + struct timeval times[2]; + times[0].tv_sec = source.atime.tv_sec; + times[0].tv_usec = source.atime.tv_nsec / 1000; + times[1].tv_sec = source.mtime.tv_sec; + times[1].tv_usec = source.mtime.tv_nsec / 1000; + updateok = (utimes(target_path.c_str(), times) == 0); + } else { + char update[1024]; + auto roles = eos_roles_opaque(); + sprintf(update, "%ceos.app=%s%s%s&mgm.pcmd=utimes" + "&tv1_sec=%llu&tv1_nsec=%llu" + "&tv2_sec=%llu&tv2_nsec=%llu", + (target.opaque.length()) ? '&' : '?', + getenv("EOSAPP") ? getenv("EOSAPP") : "eoscp", + roles.size() ? "&" : "", + roles.size() ? roles.c_str() : "", + (unsigned long long) source.atime.tv_sec, + (unsigned long long) source.atime.tv_nsec, + (unsigned long long) source.mtime.tv_sec, + (unsigned long long) source.mtime.tv_nsec); + XrdOucString request = target_path.c_str(); + request += update; + char value[4096]; + value[0] = 0; + long long update_rc = XrdPosixXrootd::QueryOpaque(request.c_str(), + value, 4096); + updateok = (update_rc >= 0); + + // Parse the stat output + if (updateok) { + char tag[1024]; + int tmp_retc; + int items = sscanf(value, "%1023s retc=%d", tag, &tmp_retc); + updateok = ((items == 2) && (strcmp(tag, "utimes:") == 0)); + } + } + + if (!updateok) { + std::cerr << "warning: creation/modification time could not be " + << "preserved for path=" << target_path.c_str() + << std::endl; + } + } + + // Verify checksum + if ((checksums) && (target.protocol != Protocol::LOCAL)) { + XrdOucString address = serveruri.c_str(); + address += "//dummy"; + XrdCl::URL url(address.c_str()); + + if (!url.IsValid()) { + std::cerr << "error: invalid file system URL=" << url.GetURL() + << " [attempting checksum]" << std::endl; + global_retc = EINVAL; + return -1; + } + + auto* fs = new XrdCl::FileSystem(url); + + if (!fs) { + std::cerr << "error: failed to get new FS object " + << "[attempting checksum]" << std::endl; + global_retc = EINVAL; + return -1; + } + + XrdCl::Buffer arg; + XrdCl::Buffer* response = nullptr; + XrdCl::XRootDStatus status; + std::string query_path = dest.c_str(); + std::string::size_type pos = query_path.rfind("//"); + + if (pos != std::string::npos) { + query_path.erase(0, pos + 1); + } + + arg.FromString(query_path); + status = fs->Query(XrdCl::QueryCode::Checksum, arg, response); + + if (status.IsOK()) { + XrdOucString xsum = response->GetBuffer(); + xsum.replace("eos ", ""); + std::cout << "path=" << source.name.c_str() << " size=" + << source.size << " checksum=" << xsum.c_str() + << std::endl; + } else { + std::cout << "warning: failed getting checksum for path=" + << source.name.c_str() << " size=" << source.size + << std::endl; + } + + delete response; + delete fs; + } + } else { + XrdOucString ssize1, ssize2; + std::cerr << "error: file size difference between source and target " + << "file source=" << source.name.c_str() << " [" + << eos::common::StringConversion::GetReadableSizeString( + ssize1, source.size, "B") + << "] target=" << target_path.c_str() << " [" + << eos::common::StringConversion::GetReadableSizeString( + ssize2, (unsigned long long) buf.st_size, "B") + << "]" << std::endl; + lrc |= 0xffff00; + } + } else { + std::cerr << "error: target file not created source=" + << source.name.c_str() << " target=" << target_path.c_str() + << std::endl; + lrc |= 0xffff00; + } + } + + // Attempt to upload temporary file + if (temporary_file) { + if (target.protocol == Protocol::GSIFTP) { + cmdtext = "globus-url-copy file://"; + cmdtext += dest.c_str(); + cmdtext += " "; + cmdtext += target_path.c_str(); + + if (silent || noprogress) { + cmdtext += " >& /dev/null"; + } + + if (debug) { + std::cerr << "[eos-cp] running: " << cmdtext.c_str() << std::endl; + } + + int rc = system(cmdtext.c_str()); + + if (WEXITSTATUS(rc)) { + std::cerr << "error: failed to upload " << target_path.c_str() + << " [protocol=gsiftp]" << std::endl; + lrc |= 0xffff00; + } + } + + if ((target.protocol == Protocol::HTTP) || + (target.protocol == Protocol::HTTPS)) { + std::cerr << "error: file uploads not supported for " + << protocol_to_string(target.protocol) << " protocol [path=" + << target_path.c_str() << "]" << std::endl; + lrc |= 0xffff00; + } + + // Clean-up the temporary file + unlink(dest.c_str()); + } + + if (!WEXITSTATUS(lrc)) { + files_copied++; + copiedsize += source.size; + } + + retc |= lrc; + } + + // Mark end timestamp + gettimeofday(&end_time, &tz); + + if (debug || !silent) { + float time_elapsed = (float)(((end_time.tv_sec - start_time.tv_sec) * 1000000 + + (end_time.tv_usec - start_time.tv_usec)) / 1000000.0); + unsigned long long copyrate = (copiedsize / time_elapsed); + XrdOucString ssize1, ssize2; + std::cerr << ((retc) ? "#WARNING " : "") + << "[eos-cp] copied " << files_copied << "/" + << (int) source_list.size() << " files and " + << eos::common::StringConversion::GetReadableSizeString(ssize1, + copiedsize, "B") + << " in " << std::fixed << std::setprecision(2) << time_elapsed + << " seconds with " + << eos::common::StringConversion::GetReadableSizeString(ssize2, + copyrate, "B/s") + << std::endl; + } + + global_retc = WEXITSTATUS(retc); + return global_retc; +} + +/* eos cp command - entry point for com_cat and legacy callers */ +int com_cp(char* argin) +{ + std::vector args = TokenizeCpArgs(argin); + CpOptions opts; + if (!ParseCpArgs(args, opts)) + return com_cp_usage(); + return cp_impl(opts); +} + +// ---------------------------------------------------------------------------- +// Helper functions implementation +// ---------------------------------------------------------------------------- + +/** + * Convenience function to be used by 'eos cp' to query EOS for file names. + * The output of the command is placed into the result vector. + * @param cmdline the eos command to be executed + * @param result reference to the result vector + * @return error code of the command + */ +int run_eos_command(const char* cmdline, std::vector& result) +{ + XrdOucString cmd = "eos -b "; + + if (user_role.length() && group_role.length()) { + cmd += "--role "; + cmd += user_role; + cmd += " "; + cmd += group_role; + cmd += " "; + } + + cmd += cmdline; + return run_command(cmd.c_str(), result); +} + +/** + * Convenience function to be used by 'eos cp' to execute a command. + * The output of the command is placed into the result vector. + * @param cmdline the bash command to be executed + * @param result reference to the result vector + * @return error code of the command + */ +int run_command(const char* cmdline, std::vector& result) +{ + FILE* fp = popen(cmdline, "r"); + char line[4096]; + int rc; + + if (!fp) { + std::cerr << "error: failed executing command " << cmdline << std::endl; + return errno; + } + + while (fgets(line, sizeof(line), fp)) { + int size = strlen(line); + + if (line[size - 1] == '\n') { + line[size - 1] = '\0'; + } + + result.emplace_back(line); + } + + rc = pclose(fp); + return WEXITSTATUS(rc); +} + +/** + * Converts from local to absolute path. + * This function makes the distinction between local or EOS paths. + * Any other protocol will be left untouched. + * Function is aware of interactive eos shell environment. + * Local files will have the 'file:' prefix removed. + * @param path the given path + * @return abspath the absolute path + */ +const char* absolute_path(const char* path) +{ + Protocol protocol = get_protocol(path); + + if (protocol != Protocol::EOS && protocol != Protocol::LOCAL) { + return strdup(path); + } + + if (strcmp(path, "-") == 0) { + return strdup(path); + } + + XrdOucString spath = path; + + if (protocol == Protocol::LOCAL && spath.beginswith("file:")) { + spath.erase(0, 5); + } + + if (!spath.beginswith("/")) { + XrdOucString abspath = ""; + + if (interactive) { + // Construct absolute path within eos shell + abspath.insert(gPwd.c_str(), 0); + } else { + // Construct absolute path within regular shell + abspath.insert("/", 0); + abspath.insert(getenv("PWD"), 0); + } + + spath.insert(abspath.c_str(), 0); + } + + // Note: eos::common::Path expects an absolute path! + // Note: eos::common::Path removes trailing '/'! + std::string trailing_slash = ""; + + if ((spath.endswith("/")) && (!spath.endswith("/./")) && + (!spath.endswith("/../"))) { + trailing_slash = "/"; + } + + // Sanitize '.' and '..' entries + spath = eos::common::Path(spath.c_str()).GetFullPath().c_str(); + spath += trailing_slash.c_str(); + return strdup(spath.c_str()); +} + +/** + * Given a symlink path of the following format 'link -> target', + * will return the name of the 'link'. + * @param path the path to check + * @return path the processed symlink name + */ +XrdOucString process_symlink(XrdOucString path) +{ + int pos = path.find(" -> "); + + if (pos != STR_NPOS) { + path.erase(pos); + } + + return path; +} + +/** + * Will check whether the given path is a directory or not. + * For local and EOS protocols, stat information is used. + * The stat structure may be passed, otherwise it is constructed. + * Function is aware of interactive eos shell environment. + * @param path the path to check + * @param protocol the protocol to access the path + * @param buf stat structure + * @return true if directory, false otherwise + */ +bool is_dir(const char* path, Protocol protocol, struct stat* buf) +{ + if (protocol != Protocol::EOS && protocol != Protocol::LOCAL) { + XrdOucString spath = path; + return spath.endswith("/"); + } + + int rc = 0; + struct stat tmpbuf {}; + + if (buf == nullptr) { + buf = &tmpbuf; + const char* abs_path = absolute_path(path); + rc = do_stat(abs_path, protocol, *buf); + free(const_cast(abs_path)); + } + + return (rc == 0) ? S_ISDIR(buf->st_mode) : false; +} + +/** + * Returns eos roles opaque info from the global user variables. + * @return roles opaque info containing eos roles + */ +std::string +eos_roles_opaque() +{ + std::string roles; + + if (user_role.length() && group_role.length()) { + roles = "eos.ruid="; + roles += user_role.c_str(); + roles += "&eos.rgid="; + roles += group_role.c_str(); + return roles; + } + + return roles; +} + +/** + * Perform stat on a given path. + * Function makes the distinction between local or EOS paths. + * @param path the path to stat + * @param protocol the protocol to access the path + * @param buf stat structure to fill + * @return rc stat error code + */ +int do_stat(const char* path, Protocol protocol, struct stat& buf) +{ + const char* abs_path = absolute_path(path); + int rc = -1; + + if (protocol == Protocol::EOS || protocol == Protocol::XROOT) { + // Stat EOS file + XrdOucString url = abs_path; + std::string roles = eos_roles_opaque(); + + // Expand '/eos/' shortcut for EOS protocol + if (url.beginswith("/eos/")) { + url = serveruri.c_str(); + url += (!url.endswith("/")) ? "/" : ""; + url += abs_path; + } + + if (!roles.empty()) { + url += (url.find("?") == STR_NPOS) ? "?" : "&"; + url += roles.c_str(); + } + + rc = XrdPosixXrootd::Stat(url.c_str(), &buf); + } else if (protocol == Protocol::LOCAL) { + // Stat local file + rc = stat(abs_path, &buf); + } + + free((char*)abs_path); + return rc; +} + +/** + * Given an S3 path, will parse and remove the opaque info. + * The following environment variables are set: + * S3_ACCESS_KEY_ID
+ * S3_SECRET_ACCESS_KEY
+ * S3_HOSTNAME
+ * @param path the S3 path + * @param opaque the opaque info to parse for S3 info + * @return url the S3 url + */ +const char* setup_s3_environment(XrdOucString path, XrdOucString opaque) +{ + XrdOucString sprot, hostport; + XrdOucString url = eos::common::StringConversion::ParseUrl(path.c_str(), + sprot, hostport); + + if (!url.length()) { + std::cerr << "error: could not parse S3 url=" << path.c_str() + << std::endl; + global_retc = EINVAL; + return 0; + } + + if (opaque.length()) { + XrdOucEnv env(opaque.c_str()); + + // Extract opaque S3 tags if present + if (env.Get("s3.id")) { + setenv("S3_ACCESS_KEY_ID", env.Get("s3.id"), 1); + } + + if (env.Get("s3.key")) { + setenv("S3_SECRET_ACCESS_KEY", env.Get("s3.key"), 1); + } + } + + if (hostport.length()) { + setenv("S3_HOSTNAME", hostport.c_str(), 1); + } + + // Apply the ROOT compatibility environment variables + if (getenv("S3_ACCESS_ID")) { + setenv("S3_ACCESS_KEY_ID", getenv("S3_ACCESS_ID"), 1); + } + + if (getenv("S3_ACCESS_KEY")) { + setenv("S3_SECRET_ACCESS_KEY", getenv("S3_ACCESS_KEY"), 1); + } + + // Check S3 environment + if ((!getenv("S3_HOSTNAME")) || (!getenv("S3_ACCESS_KEY_ID")) || + (!getenv("S3_SECRET_ACCESS_KEY"))) { + std::cerr << "error: S3 environment not set up for " << path.c_str() + << std::endl; + std::cerr << "You have to set the following environment variables: " + << "S3_ACCESS_KEY_ID or S3_ACCESS_ID\n" + << "S3_SECRET_ACCESS_KEY or S3_ACCESS_KEY\n" + << "S3_HOSTNAME (or use path with URI)" << std::endl; + global_retc = EINVAL; + return 0; + } + + return url.c_str(); +} + +/** + * Check if required tools are available to access the given path. + * @param path the path to access + */ +int check_protocol_tool(const char* path) +{ + Protocol protocol = get_protocol(path); + std::string tool = ""; + char cmd[128]; + + if (protocol == Protocol::HTTP || protocol == Protocol::HTTPS) { + tool = "curl"; + } else if (protocol == Protocol::AS3 || protocol == Protocol::S3) { + tool = "s3"; + } else if (protocol == Protocol::GSIFTP) { + tool = "globus-url-copy"; + } else { + return 0; + } + + sprintf(cmd, "which %s >& /dev/null", tool.c_str()); + int rc = system(cmd); + + if (WEXITSTATUS(rc)) { + std::cerr << "error: " << tool << " executable not found in PATH" + << std::endl; + + if (tool == "s3") { + std::cerr << " error: please install S3 executable from libs3" + << std::endl; + } + + global_retc = WEXITSTATUS(rc); + } + + return WEXITSTATUS(rc); +} + +/** + * Returns the protocol for a given path. + * Function is aware of interactive eos shell environment. + */ +Protocol get_protocol(XrdOucString path) +{ + if (path.beginswith("/eos/")) { + return Protocol::EOS; + } else if (path.beginswith("http://")) { + return Protocol::HTTP; + } else if (path.beginswith("https://")) { + return Protocol::HTTPS; + } else if (path.beginswith("gsiftp://")) { + return Protocol::GSIFTP; + } else if (path.beginswith("root://")) { + return Protocol::XROOT; + } else if (path.beginswith("as3:")) { + return Protocol::AS3; + } else if (path.beginswith("s3://")) { + return Protocol::S3; + } else if (path.beginswith("file:")) { + return Protocol::LOCAL; + } else if (path.beginswith("/") || (path.find(":/") == STR_NPOS)) { + return (interactive) ? Protocol::EOS : Protocol::LOCAL; + } + + return Protocol::UNKNOWN; +} + +/** + * Returns a string representation of the protocol. + */ +const char* protocol_to_string(Protocol protocol) +{ + if (protocol == Protocol::EOS) { + return "eos"; + } else if (protocol == Protocol::HTTP) { + return "http"; + } else if (protocol == Protocol::HTTPS) { + return "https"; + } else if (protocol == Protocol::GSIFTP) { + return "gsiftp"; + } else if (protocol == Protocol::XROOT) { + return "root"; + } else if (protocol == Protocol::AS3) { + return "as3"; + } else if (protocol == Protocol::S3) { + return "s3"; + } else if (protocol == Protocol::LOCAL) { + return "local"; + } + + return "unknown"; +} + +// ---------------------------------------------------------------------------- +// Native command registration (keeps com_cp interface for cat and other callers) +// ---------------------------------------------------------------------------- +namespace { +class CpCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "cp"; + } + const char* + description() const override + { + return "Copy files"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + (void)ctx; + if (args.empty() || wants_help(args[0].c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + CpOptions opts; + if (!ParseCpArgs(args, opts)) { + global_retc = EINVAL; + return 0; + } + return cp_impl(opts); + } + void + printHelp() const override + { + std::cerr << MakeCpHelp(); + } +}; +} // namespace + +void +RegisterCpNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} + +/* eos cat command - just eos cp with '-' destination */ +int com_cat(char* argin) +{ + std::string catarg=(const char*)argin; + catarg += " -"; + return com_cp((char*)catarg.c_str()); +} diff --git a/console/commands/native/daemon-native.cc b/console/commands/native/daemon-native.cc new file mode 100644 index 0000000000..03512ba688 --- /dev/null +++ b/console/commands/native/daemon-native.cc @@ -0,0 +1,808 @@ +// ---------------------------------------------------------------------- +// File: daemon-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include +// Native implementation adapted from legacy com_daemon +#include "common/Config.hh" +#include "common/Path.hh" +#include "common/StringTokenizer.hh" +#include "common/SymKeys.hh" +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include +#include +#include +#include +#include +#include + +// Ptrace macros compatibility +#ifdef __APPLE__ +#define EOS_PTRACE_TRACEME PT_TRACEME +#define EOS_PTRACE_ATTACH PT_ATTACH +#define EOS_PTRACE_DETACH PT_DETACH +#else +#define EOS_PTRACE_TRACEME PTRACE_TRACEME +#define EOS_PTRACE_ATTACH PTRACE_ATTACH +#define EOS_PTRACE_DETACH PTRACE_DETACH +#endif //__APPLE__ + +// Native port of legacy com_daemon +static int +native_com_daemon(char* arg) +{ +#ifdef __APPLE__ + fprintf(stderr, "error: daemon command is not support on OSX\n"); + global_retc = EINVAL; + return (0); +#else + eos::common::StringTokenizer subtokenizer(arg); + eos::common::Config cfg; + XrdOucString option = ""; + XrdOucString name = ""; + XrdOucString service = ""; + XrdOucString subcmd; + XrdOucString modules; + std::vector mods; + std::string chapter; + std::string executable; + std::string pidfile; + std::string envfile; + std::string cfile; + bool ok = false; + subtokenizer.GetLine(); + + if (wants_help(arg)) { + goto com_daemon_usage; + } + + option = subtokenizer.GetToken(); + + if (!option.length()) { + goto com_daemon_usage; + } + + if (option == "sss") { + subcmd = subtokenizer.GetToken(); + + if (!subcmd.length()) { + goto com_daemon_usage; + } + + if (subcmd == "recreate") { + if (geteuid()) { + std::cerr << "error: you have to run this command as root!" + << std::endl; + global_retc = EPERM; + return (0); + } + + struct stat buf; + + std::cerr + << "info: you are going to (re-)create the instance sss key. A " + "previous key will be moved to /etc/eos.keytab." + << std::endl; + + if (!::stat("/etc/eos.keytab", &buf)) { + std::string oldkeytab = "/etc/eos.keytab."; + oldkeytab += std::to_string(time(NULL)); + + if (::rename("/etc/eos.keytab", oldkeytab.c_str())) { + std::cerr << "error: renaming of existing old keytab file " + "/etc/eos.keytab failed!" + << std::endl; + global_retc = errno; + return (0); + } + } + + bool interactive = false; + + if (isatty(STDOUT_FILENO)) { + interactive = true; + } + + if (!interactive || ICmdHelper::ConfirmOperation()) { + if (!::stat("/opt/eos/xrootd/bin/xrdsssadmin", &buf)) { + system("yes | /opt/eos/xrootd/bin/xrdsssadmin -u daemon -g daemon -k " + "eosmaster add /etc/eos.keytab"); + system("yes | /opt/eos/xrootd/bin/xrdsssadmin -u eosnobody -g " + "eosnobody -k eosnobody add /etc/eos.keytab"); + } else { + system("yes | xrdsssadmin -u daemon -g daemon -k eosmaster add " + "/etc/eos.keytab"); + system("yes | xrdsssadmin -u eosnobody -g eosnobody -k eosnobody add " + "/etc/eos.keytab"); + } + + system("mkdir -p /etc/eos/; cat /etc/eos.keytab | grep eosnobody > " + "/etc/eos/fuse.sss.keytab; chmod 400 /etc/eos/fuse.sss.keytab"); + std::cerr << "info: recreated /etc/eos.keytab /etc/eos/fuse.sss.keytab" + << std::endl; + } else { + global_retc = EINVAL; + return (0); + } + + return (0); + } + } + + if (option == "seal") { + XrdOucString toseal = subtokenizer.GetToken(); + XrdOucString sealed; + std::string key; + + if (!toseal.length()) { + return (0); + } + + if (toseal.beginswith("/")) { + // treat it as a file + std::string contents; + eos::common::StringConversion::LoadFileIntoString(toseal.c_str(), + contents); + toseal = contents.c_str(); + } + + const char* pkey = subtokenizer.GetToken(); + + if (!pkey) { + key = eos::common::StringConversion::StringFromShellCmd( + "cat /etc/eos.keytab | grep u:daemon | md5sum"); + } else { + key = pkey; + } + + std::string shakey = eos::common::SymKey::HexSha256(key); + eos::common::SymKey::SymmetricStringEncrypt(toseal, sealed, + (char*)shakey.c_str()); + fprintf(stderr, "enc:%s\n", sealed.c_str()); + return (0); + } + + if ((option != "run") && (option != "config") && (option != "stack") && + (option != "stop") && (option != "kill") && (option != "jwk") && + (option != "module-init")) { + goto com_daemon_usage; + } + + service = subtokenizer.GetToken(); + + if ((option != "jwk") && + ((!service.length()) || ((service != "mgm") && (service != "mq") && + (service != "fst") && (service != "qdb")))) { + goto com_daemon_usage; + } + + name = subtokenizer.GetToken(); + + if (!name.length()) { + name = service.c_str(); + } + + modules = name; + modules += ".modules"; + executable = "eos-"; + executable += service.c_str(); + envfile = "/var/run/eos/"; + envfile += executable.c_str(); + envfile += "."; + envfile += name.c_str(); + envfile += ".env"; + pidfile = "/var/run/eos/xrd."; + pidfile += service.c_str(); + pidfile += "."; + pidfile += name.c_str(); + pidfile += ".pid"; + cfg.Load("generic", "all"); + ok |= cfg.ok(); + cfg.Load(service.c_str(), name.c_str(), false); + ok |= cfg.ok(); + // this might fail if there are no modules + cfg.Load(service.c_str(), modules.c_str(), false); + { + // load all the modules: + eos::common::StringConversion::Tokenize(cfg.Dump("modules", true), mods, + "\n"); + + for (size_t i = 0; i < mods.size(); ++i) { + if (mods[i].front() == '#') { + // ignore comments + continue; + } + + if (mods[i].find(" ") != std::string::npos) { + fprintf(stderr, "warning: ignoring module line '%s' (contains space)\n", + mods[i].c_str()); + continue; + } + + if (mods[i].empty()) { + // ignore empty lines + continue; + } + + if (!cfg.Load("modules", mods[i].c_str(), false)) { + fprintf(stderr, "error: failed to load module '%s'\n", mods[i].c_str()); + global_retc = EINVAL; + return 0; + } + } + } + chapter = service.c_str(); + chapter += ":xrootd:"; + chapter += name.c_str(); + cfile = "/var/run/eos/xrd.cf."; + cfile += name.c_str(); + + if (option == "config") { + char** const envv = cfg.Env("sysconfig"); + + for (size_t i = 0; i < 1024; ++i) { + if (envv[i]) { + putenv(envv[i]); + fprintf(stderr, "[putenv] %s\n", envv[i]); + } else { + break; + } + } + + if (service == "qdb") { + XrdOucString subcmd = subtokenizer.GetToken(); + + if (subcmd == "coup") { + std::string kline; + kline = + "export REDISCLI_AUTH=`cat /etc/eos.keytab`; redis-cli -p `cat "; + kline += cfile; + kline += "|grep xrd.port | cut -d ' ' -f 2` <<< raft-attempt-coup"; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } else if (subcmd == "info") { + std::string kline; + kline = + "export REDISCLI_AUTH=`cat /etc/eos.keytab`; redis-cli -p `cat "; + kline += cfile; + kline += "|grep xrd.port | cut -d ' ' -f 2` <<< raft-info"; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } else if (subcmd == "remove") { + XrdOucString member = subtokenizer.GetToken(); + + if (!member.length()) { + fprintf(stderr, "error: remove misses member argument host:port : " + "'eos daemon config qdb qdb remove host:port'\n"); + global_retc = EINVAL; + return 0; + } else { + std::string kline; + kline = + "export REDISCLI_AUTH=`cat /etc/eos.keytab`; redis-cli -p `cat "; + kline += cfile; + kline += + "|grep xrd.port | cut -d ' ' -f 2` <<< \"raft-remove-member "; + kline += member.c_str(); + kline += "\""; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } + } else if (subcmd == "add") { + XrdOucString member = subtokenizer.GetToken(); + if (!member.length()) { + fprintf(stderr, "error: add misses member argument host:port : 'eos " + "daemon config qdb qdb add host:port'\n"); + global_retc = EINVAL; + return 0; + } else { + std::string kline; + kline = + "export REDISCLI_AUTH=`cat /etc/eos.keytab`; redis-cli -p `cat "; + kline += cfile; + kline += "|grep xrd.port | cut -d ' ' -f 2` \"<<< raft-add-observer "; + kline += member.c_str(); + kline += "\""; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } + } else if (subcmd == "promote") { + XrdOucString member = subtokenizer.GetToken(); + + if (!member.length()) { + fprintf(stderr, "error: promote misses member argument host:port : " + "'eos daemon config qdb qdb promote host:port'\n"); + global_retc = EINVAL; + return 0; + } else { + std::string kline; + kline = + "export REDISCLI_AUTH=`cat /etc/eos.keytab`; redis-cli -p `cat "; + kline += cfile; + kline += + "|grep xrd.port | cut -d ' ' -f 2` <<< \"raft-promote-observer "; + kline += member.c_str(); + kline += "\""; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } + } else if (subcmd == "new") { + XrdOucString member = subtokenizer.GetToken(); + if (member != "observer") { + fprintf(stderr, "error: new misses 'observer' arguement : 'eos " + "daemon config qdb qdb new observer'\n"); + global_retc = EINVAL; + return 0; + } else { + std::string stopqdb = "systemctl stop qdb "; + stopqdb += name.c_str(); + system(stopqdb.c_str()); + + std::string qdbpath = getenv("QDB_PATH") ? getenv("QDB_PATH") : ""; + std::string qdbcluster = + getenv("QDB_CLUSTER_ID") ? getenv("QDB_CLUSTER_ID") : ""; + std::string qdbnode = getenv("QDB_NODE") ? getenv("QDB_NODE") : ""; + if (qdbpath.empty()) { + fprintf(stderr, + "error: QDB_PATH is undefined in your configuration\n"); + global_retc = EINVAL; + return 0; + } + if (qdbcluster.empty()) { + fprintf( + stderr, + "error: QDB_CLUSTER_ID is undefined in your configuration\n"); + global_retc = EINVAL; + return 0; + } + if (qdbnode.empty()) { + fprintf(stderr, + "error: QDB_NODE is undefined in your configuration\n"); + global_retc = EINVAL; + return 0; + } + struct stat buf; + if (!::stat(qdbpath.c_str(), &buf)) { + fprintf(stderr, + "error: path '%s' exists - to create a new observer this " + "path has to be changed or removed\n", + qdbpath.c_str()); + global_retc = EINVAL; + return 0; + } else { + fprintf(stderr, "info: creating QDB under %s ...\n", + qdbpath.c_str()); + } + + std::string kline; + kline = "quarkdb-create --path "; + kline += qdbpath; + kline += " --clusterID "; + kline += qdbcluster; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + if (!global_retc) { + fprintf(stderr, + "info: to get this node joining the cluster you do:\n"); + fprintf(stderr, "1 [ this node ] : systemctl start eos5-@qdb@%s\n", + name.c_str()); + fprintf(stderr, + "2 [ leader ] : eos daemon config qdb %s add %s\n", + name.c_str(), qdbnode.c_str()); + fprintf(stderr, + "3 [ leader ] : eos daemon config qdb %s promote %s\n", + name.c_str(), qdbnode.c_str()); + } + return (0); + } + } else if (subcmd == "backup") { + std::string qdbpath = "/var/lib/qdb1"; + + for (auto it : cfg[chapter.c_str()]) { + size_t pos; + + if ((pos = it.find("redis.database")) != std::string::npos) { + qdbpath = it; + qdbpath.erase(pos, 15); + } + } + + std::string kline; + std::string qdblocation = qdbpath; + qdblocation += "/backup/"; + qdblocation += std::to_string(time(NULL)); + kline += + "export REDISCLI_AUTH=`cat /etc/eos.keytab`; redis-cli -p `cat "; + kline += cfile; + kline += "|grep xrd.port | cut -d ' ' -f 2` <<< \"quarkdb-checkpoint "; + kline += qdblocation; + kline += "\""; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), + WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } + } + + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- i n i t -----------------\n"); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "%s\n", cfg.Dump("init", true).c_str()); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- s y s c o n f i g -------\n"); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "%s\n", cfg.Dump("sysconfig", true).c_str()); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- m o d u l e s -----------\n"); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "%s\n", cfg.Dump("modules", true).c_str()); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- x r o o t d ------------\n"); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# running config file: %s\n", cfile.c_str()); + fprintf(stderr, "%s\n", cfg.Dump(chapter.c_str(), true).c_str()); + fprintf(stderr, "#########################################\n"); + } else if (option == "module-init") { + std::string initfile = "/tmp/.eos.daemon.init"; + std::string initsection = name.c_str(); + initsection += ":init"; + + if (!eos::common::StringConversion::SaveStringIntoFile( + initfile.c_str(), cfg.Dump(initsection.c_str(), true))) { + fprintf(stderr, "error: unable to create startup config file '%s'\n", + initfile.c_str()); + global_retc = errno; + return (0); + } else { + chmod(initfile.c_str(), S_IRWXU); + // run the init file + int rc = system(initfile.c_str()); + + if (WEXITSTATUS(rc)) { + fprintf(stderr, "error: init script '%s' failed with errc=%d\n", + initsection.c_str(), WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } + } + } else if (option == "stack") { + std::string kline; + kline = "test -e "; + kline += envfile.c_str(); + kline += " && eu-stack -p `cat "; + kline += envfile.c_str(); + kline += "| cut -d '&' -f 1 | cut -d '=' -f 2`"; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } else if (option == "jwk") { + XrdOucString jwkfile = name.c_str(); + struct stat buf; + if (::stat(jwkfile.c_str(), &buf)) { + fprintf(stderr, "error: jwk key file '%s' does not exist!\n", + jwkfile.c_str()); + global_retc = ENOENT; + return (0); + } + std::string kline; + kline = "env EOS_JWK=\"$(cat \""; + kline += jwkfile.c_str(); + kline += "\")\" /sbin/eos-jwk-https"; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } else if (option == "kill") { + std::string kline; + kline = "test -e "; + kline += envfile.c_str(); + kline += " && kill -9 `cat "; + kline += envfile.c_str(); + kline += "| cut -d '&' -f 1 | cut -d '=' -f 2`"; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } else if (option == "stop") { + std::string kline; + kline = "test -e "; + kline += envfile.c_str(); + kline += " && kill -15 `cat "; + kline += envfile.c_str(); + kline += "| cut -d '&' -f 1 | cut -d '=' -f 2`"; + int rc = system(kline.c_str()); + fprintf(stderr, "info: run '%s' retc=%d\n", kline.c_str(), WEXITSTATUS(rc)); + global_retc = WEXITSTATUS(rc); + return (0); + } else if (option == "run") { + if (!cfg.Has(chapter.c_str())) { + fprintf(stderr, + "error: missing service configuration [%s] in generic config " + "file '/etc/eos/config/generic/all' or '/etc/eos/config/%s/%s'\n", + chapter.c_str(), service.c_str(), name.c_str()); + global_retc = EINVAL; + return (0); + } + + char** const envv = cfg.Env("sysconfig"); + + for (size_t i = 0; i < 1024; ++i) { + if (envv[i]) { + fprintf(stderr, "%s\n", envv[i]); + } else { + break; + } + } + + if (cfg.Has("init")) { + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- i n i t -----------------\n"); + fprintf(stderr, "# ---------------------------------------\n"); + + if (cfg.Has("unshare")) { + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- u n s h a r e -----------\n"); + + if (unshare(CLONE_NEWNS)) { + fprintf(stderr, + "warning: failed to unshare mount namespace errno=%d\n", + errno); + } + + if (mount("none", "/", NULL, MS_REC | MS_PRIVATE, NULL)) { + fprintf(stderr, "warning: failed none mount / - errno=%d\n", errno); + } + + fprintf(stderr, "# ---------------------------------------\n"); + } + + for (auto it : cfg["init"]) { + bool exit_on_failure = false; + fprintf(stderr, "# run: %s\n", it.c_str()); + pid_t pid; + std::string cline = it; + + if (cline.substr(0, 4) == "enc:") { + std::string key; + const char* pkey = subtokenizer.GetToken(); + + if (!pkey) { + key = eos::common::StringConversion::StringFromShellCmd( + "cat /etc/eos.keytab | grep u:daemon | md5sum"); + } else { + key = pkey; + } + + std::string shakey = eos::common::SymKey::HexSha256(key); + XrdOucString in = cline.substr(4).c_str(); + ; + XrdOucString out; + eos::common::SymKey::SymmetricStringDecrypt(in, out, + (char*)shakey.c_str()); + + if (!out.c_str()) { + fprintf(stderr, "error: encoded init line '%s' cannot be decoded\n", + in.c_str()); + continue; + } + + cline = out.c_str(); + exit_on_failure = true; + } + + if (exit_on_failure) { + // test that nobody traces us .. + if (!(pid = fork())) { + pause(); + exit(0); + } else { + if (ptrace(EOS_PTRACE_ATTACH, pid, 0, 0)) { + kill(pid, SIGKILL); + fprintf(stderr, + "error: failed to attach to forked process pid=%d " + "errno=%d - we are untraceable\n", + pid, errno); + + if (exit_on_failure) { + exit(-1); + } + } else { + ptrace(EOS_PTRACE_DETACH, pid, 0, 0); + kill(pid, 9); + waitpid(pid, 0, 0); + } + } + } + + if (!(pid = fork())) { + execle("/bin/bash", "eos-bash", "-c", cline.c_str(), NULL, envv); + exit(0); + } else { + waitpid(pid, 0, 0); + } + } + } + + if (ok) { + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# ------------- x r o o t d ------------\n"); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "# running config file: %s\n", cfile.c_str()); + fprintf(stderr, "# ---------------------------------------\n"); + fprintf(stderr, "%s\n", cfg.Dump(chapter.c_str(), true).c_str()); + fprintf(stderr, "#########################################\n"); + eos::common::Path cPath(cfile.c_str()); + + if (!cPath.MakeParentPath(0x1ed)) { + fprintf(stderr, "error: unable to create run directory '%s'\n", + cPath.GetParentPath()); + global_retc = errno; + return (0); + } + + if (!eos::common::StringConversion::SaveStringIntoFile( + cfile.c_str(), cfg.Dump(chapter.c_str(), true))) { + fprintf(stderr, "error: unable to create startup config file '%s'\n", + cfile.c_str()); + global_retc = errno; + return (0); + } + + ::chdir(cPath.GetParentPath()); + std::string logfile = "/var/log/eos/xrdlog."; + logfile += service.c_str(); + + if (service == "qdb") { + execle("/opt/eos/xrootd/bin/xrootd", executable.c_str(), "-n", + name.c_str(), "-c", cfile.c_str(), "-l", logfile.c_str(), "-R", + "daemon", "-k", "fifo", "-s", pidfile.c_str(), NULL, envv); + } else { + execle("/opt/eos/xrootd/bin/xrootd", executable.c_str(), "-n", + name.c_str(), "-c", cfile.c_str(), "-l", logfile.c_str(), "-R", + "daemon", "-s", pidfile.c_str(), NULL, envv); + } + + return (0); + } else { + fprintf(stderr, "error: rc=%d msg=%s\n", cfg.getErrc(), + cfg.getMsg().c_str()); + global_retc = cfg.getErrc(); + return (0); + } + } + + return (0); +com_daemon_usage: + fprintf( + stderr, + "Usage: daemon config|sss|kill|run|stack|stop|jwk|module-init " + "[name] [subcmd] : \n"); + fprintf(stderr, " := mq | mgm | fst | qdb\n"); + fprintf(stderr, " config " + " - configure a service / show configuration\n"); + fprintf(stderr, " kill " + " - kill -9 a given service\n"); + fprintf(stderr, + " run " + " - run the given service daemon optionally identified by name\n"); + fprintf(stderr, + " sss recreate " + " - re-create an instance sss key and the eosnobody keys " + "(/etc/eos.keytab,/etc/eos/fuse.sss.keytab)'\n"); + fprintf(stderr, " stack " + " - print an 'eu-stack'\n"); + fprintf(stderr, " stop " + " - kill -15 a given service\n"); + fprintf(stderr, " jwk " + " - run a 'jwk' public key server on port 4443\n"); + fprintf(stderr, " module-init " + " - run the init procedure for a module\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " examples: eos daemon config qdb qdb coup " + " - try to make instance [qdb] a leader of QDB\n"); + fprintf(stderr, " eos daemon config qdb qdb info " + " - show raft-info for the [qdb] QDB instance\n"); + fprintf(stderr, " eos daemon config qdb qdb remove host:port " + " - remove a member of the qdb cluster\n"); + fprintf(stderr, " eos daemon config qdb qdb add host:port " + " - add an observer to the qdb cluster\n"); + fprintf(stderr, + " eos daemon config qdb qdb promote host:port " + " - promote an observer to a full member of the qdb cluster\n"); + fprintf(stderr, " eos daemon config qdb qdb new observer " + " - create a new observer\n"); + fprintf(stderr, " eos daemon config fst fst.1 " + " - show the init,sysconfig and xrootd config for " + "the [fst.1] FST service\n"); + fprintf(stderr, " eos daemon kill mq " + " - shoot the MQ service with signal -9\n"); + fprintf(stderr, + " eos daemon stop mq " + " - gracefully shut down the MQ service with signal -15\n"); + fprintf(stderr, " eos daemon stack mgm " + " - take an 'eu-stack' of the MGM service\n"); + fprintf(stderr, " eos daemon run fst fst.1 " + " - run the fst.1 subservice FST\n"); + global_retc = EINVAL; + return (0); +#endif +} + +// Legacy compatibility symbol required by ConsoleMain and other modules +int +com_daemon(char* arg) +{ + return native_com_daemon(arg); +} + +namespace { +class DaemonCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "daemon"; + } + const char* + description() const override + { + return "Run EOS daemon control"; + } + bool + requiresMgm(const std::string&) const override + { + return false; + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); +#ifdef __APPLE__ + fprintf(stderr, "error: daemon command is not support on OSX\n"); + global_retc = EINVAL; + return 0; +#else + return native_com_daemon((char*)joined.c_str()); +#endif + } + void + printHelp() const override + { + } +}; +} // namespace + +void +RegisterDaemonNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/debug-cmd-native.cc b/console/commands/native/debug-cmd-native.cc new file mode 100644 index 0000000000..c1d85c51da --- /dev/null +++ b/console/commands/native/debug-cmd-native.cc @@ -0,0 +1,184 @@ +// ---------------------------------------------------------------------- +// File: debug-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include +#include +#include +#include + +namespace { +std::string MakeDebugHelp() +{ + std::ostringstream oss; + oss << "Usage: debug get|this| [node-queue] [--filter ]\n" + << "'[eos] debug ...' allows to get or set the verbosity of the EOS " + "log files in MGM and FST services.\n\n" + << "debug get : retrieve the current log level for the mgm and fsts " + "node-queue\n\n" + << "debug this : toggle EOS shell debug mode\n\n" + << "debug [--filter ] : set the MGM where the " + "console is connected to into debug level \n\n" + << "debug [--filter ] : set the " + " into debug level .\n" + << " - are internal EOS names e.g. " + "'/eos/:/fst'\n" + << " - is a comma separated list of strings of software " + "units which should be filtered out in the message log!\n\n" + << "The allowed debug levels are: " + "debug,info,warning,notice,err,crit,alert,emerg\n"; + return oss.str(); +} + +void ConfigureDebugApp(CLI::App& app, + std::string& mode, + std::string& node, + std::string& filter) +{ + app.name("debug"); + app.description("Set debug level"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeDebugHelp(); + })); + app.add_option("mode", mode, "get|this|")->required(); + app.add_option("node", node, "node-queue (e.g. /eos/host:port/fst)"); + app.add_option("--filter", filter, "comma-separated unit list to filter"); +} + +class DebugCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "debug"; + } + const char* + description() const override + { + return "Set debug level"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + std::string mode; + std::string node; + std::string filter; + ConfigureDebugApp(app, mode, node, filter); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::string cmd; + if (mode == "get" || mode == "this") { + cmd = mode; + } else { + cmd = mode; + if (!node.empty()) + cmd += " " + node; + if (!filter.empty()) + cmd += " --filter " + filter; + } + + class LocalHelper : public ICmdHelper { + public: + using ICmdHelper::ICmdHelper; + bool + ParseCommand(const char* arg) override + { + eos::console::DebugProto* debugproto = mReq.mutable_debug(); + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); + std::string token; + if (!tokenizer.NextToken(token)) { + return false; + } + if (token == "get") { + auto* get = debugproto->mutable_get(); + get->set_placeholder(true); + } else if (token == "this") { + global_debug = !global_debug; + gGlobalOpts.mDebug = global_debug; + fprintf(stderr, "info: toggling shell debugmode to debug=%d\n", + global_debug); + mIsLocal = true; + } else { + auto* set = debugproto->mutable_set(); + set->set_debuglevel(token); + if (tokenizer.NextToken(token)) { + if (token == "--filter") { + if (!tokenizer.NextToken(token)) + return false; + set->set_filter(token); + } else { + set->set_nodename(token); + if (tokenizer.NextToken(token)) { + if (token != "--filter") + return false; + else { + if (!tokenizer.NextToken(token)) + return false; + set->set_filter(token); + } + } + } + } + } + return true; + } + }; + + LocalHelper helper(gGlobalOpts); + if (!helper.ParseCommand(cmd.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeDebugHelp().c_str()); + } +}; +} // namespace + +void +RegisterDebugNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/devices-proto-native.cc b/console/commands/native/devices-proto-native.cc new file mode 100644 index 0000000000..8f278877d5 --- /dev/null +++ b/console/commands/native/devices-proto-native.cc @@ -0,0 +1,139 @@ +// ---------------------------------------------------------------------- +// File: devices-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include + +namespace { +std::string MakeDevicesHelp() +{ + return "Usage: devices ls [-l] [-m] [--refresh]\n\n" + "Print statistics per space of all storage devices based on S.M.A.R.T.\n\n" + "Options:\n" + " -l print S.M.A.R.T information for each configured filesystem\n" + " -m print monitoring output format (key=val)\n" + " --refresh force reparse of current S.M.A.R.T information\n\n" + "Use 'eos --json devices ls' for JSON output.\n"; +} + +void ConfigureDevicesApp(CLI::App& app) +{ + app.name("devices"); + app.description("Get Device Information"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeDevicesHelp(); + })); +} + +// Ported DevicesHelper from com_proto_devices.cc +class DevicesHelper : public ICmdHelper { +public: + explicit DevicesHelper(const GlobalOptions& opts) : ICmdHelper(opts) + { + mIsAdmin = true; + } + bool + ParseCommand(const char* arg) override + { + XrdOucString token; + eos::console::DevicesProto* devices = mReq.mutable_devices(); + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); + + if (!tokenizer.NextToken(token)) { + return false; + } + + eos::console::DevicesProto_LsProto* ls = devices->mutable_ls(); + ls->set_outformat(eos::console::DevicesProto_LsProto::NONE); + + if (token == "ls") { + do { + tokenizer.NextToken(token); + if (!token.length()) { + return true; + } + if (token == "-l") { + ls->set_outformat(eos::console::DevicesProto_LsProto::LISTING); + } else if (token == "-m") { + ls->set_outformat(eos::console::DevicesProto_LsProto::MONITORING); + } else if (token == "--refresh") { + ls->set_refresh(true); + } else { + return false; + } + } while (token.length()); + } else { + return false; + } + + return true; + } +}; + +class DevicesProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "devices"; + } + const char* + description() const override + { + return "Get Device Information"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + DevicesHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(true, true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureDevicesApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterDevicesProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/df-proto-native.cc b/console/commands/native/df-proto-native.cc new file mode 100644 index 0000000000..d245043198 --- /dev/null +++ b/console/commands/native/df-proto-native.cc @@ -0,0 +1,145 @@ +// ---------------------------------------------------------------------- +// File: df-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include + +namespace { +std::string MakeDfHelp() +{ + return "Usage: df [-m|-H|-b] [path]\n\n" + "Print unix-like 'df' information (1024 base).\n\n" + "Options:\n" + " -m print in monitoring format\n" + " -H print human readable in units of 1000\n" + " -b print raw bytes/number values\n"; +} + +void ConfigureDfApp(CLI::App& app) +{ + app.name("df"); + app.description("Get df output"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeDfHelp(); + })); +} + +// Ported DfHelper from com_proto_df.cc +class DfHelper : public ICmdHelper { +public: + explicit DfHelper(const GlobalOptions& opts) : ICmdHelper(opts) {} + bool + ParseCommand(const char* arg) override + { + eos::console::DfProto* dfproto = mReq.mutable_df(); + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); + std::string token; + + dfproto->set_si(true); + dfproto->set_readable(true); + + if (!tokenizer.NextToken(token)) { + return true; + } + + if (token == "-m") { + dfproto->set_monitoring(true); + dfproto->set_readable(false); + } else if (token == "-H") { + dfproto->set_si(false); + dfproto->set_readable(true); + } else if (token == "-b") { + dfproto->set_si(false); + dfproto->set_readable(false); + } else { + if (token.substr(0, 1) != "/") { + return false; + } + } + + std::string path = token; + if (tokenizer.NextToken(token)) { + if (token.substr(0, 1) == "-") { + return false; + } + if (token.substr(0, 1) != "/") { + return false; + } + path = token; + } + + if (tokenizer.NextToken(token)) { + return false; + } + dfproto->set_path(path); + return true; + } +}; + +class DfProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "df"; + } + const char* + description() const override + { + return "Get df output"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) { + oss << ' '; + } + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + DfHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureDfApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterDfProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/du-native.cc b/console/commands/native/du-native.cc new file mode 100644 index 0000000000..cb37d6f768 --- /dev/null +++ b/console/commands/native/du-native.cc @@ -0,0 +1,128 @@ +// ---------------------------------------------------------------------- +// File: du-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +namespace { +class DuCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "du"; + } + const char* + description() const override + { + return "Get du output"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + + int + run(const std::vector& args, CommandContext& ctx) override + { + IConsoleCommand* findCmd = CommandRegistry::instance().find("find"); + if (!findCmd) { + fprintf(stderr, "error: 'find' command not available\n"); + global_retc = EINVAL; + return 0; + } + + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) { + oss << ' '; + } + oss << args[i]; + } + std::string joined = oss.str(); + + if (wants_help(joined.c_str(), true)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + eos::common::StringTokenizer tokenizer(joined.c_str()); + tokenizer.GetLine(); + std::string token; + bool printfiles = false; + bool printreadable = false; + bool printsummary = false; + bool printsi = false; + std::string path; + + do { + if (!tokenizer.NextToken(token)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + if (token == "-a") { + printfiles = true; + } else if (token == "-h") { + printreadable = true; + } else if (token == "-s") { + printsummary = true; + } else if (token == "--si") { + printsi = true; + } else { + path = abspath(token.c_str()); + break; + } + } while (1); + + std::vector findArgs; + findArgs.emplace_back("--du"); + if (!printfiles) { + findArgs.emplace_back("-d"); + } + if (printsi) { + findArgs.emplace_back("--du-si"); + } + if (printreadable) { + findArgs.emplace_back("--du-h"); + } + if (printsummary) { + findArgs.emplace_back("--maxdepth"); + findArgs.emplace_back("0"); + } + findArgs.emplace_back(path); + + int rc = findCmd->run(findArgs, ctx); + global_retc = rc; + return rc; + } + + void + printHelp() const override + { + fprintf(stderr, "usage:\n" + "du [-a][-h][-s][--si] path\n" + "'[eos] du ...' print unix like 'du' information showing " + "subtreesize for directories\n" + "\n" + "Options:\n" + "\n" + "-a : print also for files\n" + "-h : print human readable in units of 1000\n" + "-s : print only the summary\n" + "--si : print in si units\n"); + } +}; +} // namespace + +void +RegisterDuNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/du-proto-native.cc b/console/commands/native/du-proto-native.cc new file mode 100644 index 0000000000..c9c082c93e --- /dev/null +++ b/console/commands/native/du-proto-native.cc @@ -0,0 +1,144 @@ +// ---------------------------------------------------------------------- +// File: du-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +namespace { +std::string MakeDuHelp() +{ + return "Usage: du [-a] [-h] [-s] [--si] path\n\n" + "Print unix-like 'du' information showing subtree size for directories.\n\n" + "Options:\n" + " -a print also for files\n" + " -h print human readable in units of 1000\n" + " -s print only the summary\n" + " --si print in SI units\n"; +} + +void ConfigureDuApp(CLI::App& app) +{ + app.name("du"); + app.description("Get du output"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeDuHelp(); + })); +} + +class DuCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "du"; + } + const char* + description() const override + { + return "Get du output"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + + int + run(const std::vector& args, CommandContext& ctx) override + { + IConsoleCommand* findCmd = CommandRegistry::instance().find("find"); + if (!findCmd) { + fprintf(stderr, "error: 'find' command not available\n"); + global_retc = EINVAL; + return 0; + } + + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) { + oss << ' '; + } + oss << args[i]; + } + std::string joined = oss.str(); + + if (wants_help(joined.c_str(), true)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + eos::common::StringTokenizer tokenizer(joined.c_str()); + tokenizer.GetLine(); + std::string token; + bool printfiles = false; + bool printreadable = false; + bool printsummary = false; + bool printsi = false; + std::string path; + + do { + if (!tokenizer.NextToken(token)) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + if (token == "-a") { + printfiles = true; + } else if (token == "-h") { + printreadable = true; + } else if (token == "-s") { + printsummary = true; + } else if (token == "--si") { + printsi = true; + } else { + path = abspath(token.c_str()); + break; + } + } while (1); + + std::vector findArgs; + findArgs.emplace_back("--du"); + if (!printfiles) { + findArgs.emplace_back("-d"); + } + if (printsi) { + findArgs.emplace_back("--du-si"); + } + if (printreadable) { + findArgs.emplace_back("--du-h"); + } + if (printsummary) { + findArgs.emplace_back("--maxdepth"); + findArgs.emplace_back("0"); + } + findArgs.emplace_back(path); + + int rc = findCmd->run(findArgs, ctx); + global_retc = rc; + return rc; + } + + void + printHelp() const override + { + CLI::App app; + ConfigureDuApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterDuNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/evict-cmd-native.cc b/console/commands/native/evict-cmd-native.cc new file mode 100644 index 0000000000..e72cdd32c5 --- /dev/null +++ b/console/commands/native/evict-cmd-native.cc @@ -0,0 +1,153 @@ +// ---------------------------------------------------------------------- +// File: evict-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include +#include "console/ConsoleMain.hh" +#include +#include + +namespace { +std::string MakeEvictHelp() +{ + std::ostringstream oss; + oss << "Usage: evict [--fsid ] [--ignore-removal-on-fst] " + "[--ignore-evict-counter] |fid:|fxid: " + "[|fid:|fxid:] ...\n" + << " Removes disk replicas of the given files, separated by space\n\n" + << "Options:\n" + << " --ignore-evict-counter : Force eviction by bypassing evict " + "counter\n" + << " --fsid : Evict disk copy only from a single " + "fsid\n" + << " --ignore-removal-on-fst : Ignore file removal on fst, " + "namespace-only operation\n\n" + << " This command requires 'write' and 'p' acl flag permission\n"; + return oss.str(); +} + +void ConfigureEvictApp(CLI::App& app, + bool& opt_ignore_evict_counter, + bool& opt_ignore_removal_on_fst, + std::string& fsid, + std::vector& paths) +{ + app.name("evict"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeEvictHelp(); + })); + app.add_flag("--ignore-evict-counter", opt_ignore_evict_counter, + "ignore evict counter"); + app.add_flag("--ignore-removal-on-fst", opt_ignore_removal_on_fst, "ns-only"); + app.add_option("--fsid", fsid, "single fsid"); + app.add_option("paths", paths); +} + +class EvictCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "evict"; + } + const char* + description() const override + { + return "Evict disk replicas of a file if it has tape replicas"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + app.allow_extras(); + bool opt_ignore_evict_counter = false; + bool opt_ignore_removal_on_fst = false; + std::string fsid; + std::vector pos; + ConfigureEvictApp(app, opt_ignore_evict_counter, opt_ignore_removal_on_fst, + fsid, pos); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (pos.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString in = "mgm.cmd=evict"; // using proto interface + // Build protobuf-like request via client_command wrapper + // Fallback to legacy: use com_evict style mgm.cmd if available + // Here, we map to mgm.cmd=evict helper expected by backend + if (opt_ignore_evict_counter) + in += "&mgm.evict.ignoreevictcounter=1"; + if (opt_ignore_removal_on_fst) + in += "&mgm.evict.ignoreremovalonfst=1"; + if (!fsid.empty()) { + in += "&mgm.evict.fsid="; + in += fsid.c_str(); + } + for (const auto& a : pos) { + XrdOucString path = a.c_str(); + unsigned long long fid = 0ull; + if (Path2FileDenominator(path, fid)) { + in += "&mgm.evict.fid="; + in += std::to_string(fid).c_str(); + } else { + XrdOucString ap = abspath(path.c_str()); + in += "&mgm.evict.path="; + in += ap; + } + } + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + bool opt_ignore_evict_counter = false; + bool opt_ignore_removal_on_fst = false; + std::string fsid; + std::vector pos; + ConfigureEvictApp(app, opt_ignore_evict_counter, opt_ignore_removal_on_fst, + fsid, pos); + const std::string help = app.help(); + std::cerr << help << std::endl; + } +}; +} // namespace + +void +RegisterEvictNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/file-cmd-native.cc b/console/commands/native/file-cmd-native.cc new file mode 100644 index 0000000000..c7e9e062de --- /dev/null +++ b/console/commands/native/file-cmd-native.cc @@ -0,0 +1,1135 @@ +// ---------------------------------------------------------------------- +// File: file-native.cc +// ---------------------------------------------------------------------- + +#define __STDC_FORMAT_MACROS +#include + +#include "common/FileId.hh" +#include +#include "common/Fmd.hh" +#include "common/LayoutId.hh" +#include "common/Logging.hh" +#include "common/StringConversion.hh" +#include "common/SymKeys.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#define ECOMM 70 +#endif + +namespace { +std::string MakeFileHelp() +{ + std::ostringstream oss; + oss << "Usage: file [args...]\n\n" + << "'[eos] file ..' provides the file management interface of EOS.\n\n"; + oss << "adjustreplica [--nodrop] [|fid:|fxid:] [space [subgroup]] [--exclude-fs ]\n" + << " Tries to bring files with replica layouts to the nominal replica level [need root].\n" + << " --exclude-fs exclude the given filesystem from being used for the replica adjustment\n\n"; + oss << "check [|fid:|fxid:] [%size%checksum%nrep%diskchecksum%force%output%silent]\n" + << " Retrieves stat information from the physical replicas and verifies correctness.\n" + << " %size return EFAULT if mismatch between the size meta data information\n" + << " %checksum return EFAULT if mismatch between the checksum meta data information\n" + << " %nrep return EFAULT if mismatch between layout number of replicas and existing replicas\n" + << " %diskchecksum return EFAULT if mismatch between disk checksum on FST and reference checksum\n" + << " %silent suppress all information for each replica to be printed\n" + << " %force force to get the MD even if the node is down\n" + << " %output print lines with inconsistency information\n\n"; + oss << "convert [|fid:|fxid:] [:||] " + << "[target-space] [placement-policy] [checksum] [--rewrite]\n" + << " Convert the layout of a file.\n" + << " : target layout and number of stripes\n" + << " hexadecimal layout id\n" + << " name of sys.conversion. in parent directory defining target layout\n" + << " optional name of target space or group e.g. default or default.3\n" + << " scattered, hybrid:, gathered:\n" + << " optional target checksum name (md5, adler, etc.)\n" + << " --rewrite run conversion rewriting the file, creating new copies and dropping old\n\n"; + oss << "copy [-f] [-s] [-c] \n" + << " Synchronous third party copy from to .\n" + << " source file or directory (|fid:|fxid:)\n" + << " destination file (if source is file) or directory\n" + << " -f force overwrite\n" + << " -s don't print output\n" + << " -c clone the file (keep ctime, mtime)\n\n"; + oss << "drop [|fid:|fxid:] [-f]\n" + << " Drop the file from . -f force removes replica without trigger/wait for deletion " + << "(used to retire a filesystem).\n\n"; + oss << "info [|fid:|fxid:|pid:|pxid:|inode:] [options]\n" + << " Show file info. Options: --path, --fid, --fxid, --size, --checksum, --fullpath, " + << "--proxy, -m, -s|--silent.\n\n"; + oss << "layout |fid:|fxid: -stripes \n" + << " Change the number of stripes of a file with replica layout to .\n" + << "layout |fid:|fxid: -checksum \n" + << " Change the checksum-type of a file to .\n" + << "layout |fid:|fxid: -type \n" + << " Change the layout-type of a file to (as shown by file info).\n\n"; + oss << "move [|fid:|fxid:] \n" + << " Move the file from to .\n\n"; + oss << "purge [purge-version]\n" + << " Keep maximum versions of a file. If not specified apply sys.versioning attribute.\n\n"; + oss << "rename [|fid:|fxid:] \n" + << " Rename from to name (works for files and directories).\n\n"; + oss << "rename_with_symlink \n" + << " Rename/move source file to destination directory atomically:\n" + << " - move file to destination directory\n" + << " - create symlink in the source directory to the new location\n\n"; + oss << "replicate [|fid:|fxid:] \n" + << " Replicate file part on to .\n\n"; + oss << "share [lifetime]\n" + << " Create a share link. defaults to 28d (1, 1s, 1d, 1w, 1mo, 1y, ...).\n\n"; + oss << "symlink [-f] \n" + << " Create a symlink with pointing to . -f force overwrite.\n\n"; + oss << "tag |fid:|fxid: +|-|~\n" + << " Add/remove/unlink a filesystem location to/from a file in the location index " + << "(does not move any data).\n" + << " Unlink keeps the location in the list of deleted files (gets a deletion request).\n\n"; + oss << "touch [-a] [-n] [-0] |fid:|fxid: [linkpath|size [hexchecksum]]\n" + << " Create/touch a 0-size/0-replica file if does not exist or update mtime of existing file.\n" + << " -n disable placement logic (default uses placement)\n" + << " -0 truncate a file\n" + << " -a absorb (adopt) a file from hardlink path - file disappears from given path and is taken under EOS FST control\n" + << " linkpath hard- or softlink the touched file to a shared filesystem\n" + << " size preset the size for a new touched file\n" + << " hexchecksum checksum information for a new touched file\n" + << "touch -l |fid:|fxid: [ [=user|app]]\n" + << " Touch and create an extended attribute lock with (default 24h).\n" + << " relaxes lock owner: same user or same app (default: both must match).\n" + << " EBUSY if lock held by another; second call by same caller extends lifetime.\n" + << " Use with 'eos -a application' to tag a client with an application for the lock.\n" + << "touch -u |fid:|fxid:\n" + << " Remove an extended attribute lock. No error if no lock; EBUSY if held by someone else.\n\n"; + oss << "verify |fid:|fxid: [] [-checksum] [-commitchecksum] [-commitsize] [-commitfmd] [-rate ] [-resync]\n" + << " Verify a file against the disk images.\n" + << " verify only the replica on \n" + << " -checksum trigger checksum calculation during verification\n" + << " -commitchecksum commit the computed checksum to the MGM\n" + << " -commitsize commit the file size to the MGM\n" + << " -commitfmd commit the FMD to the MGM\n" + << " -rate restrict verification speed to per node\n" + << " -resync ask all locations to resync their file md records\n\n"; + oss << "version [purge-version]\n" + << " Create a new version of a file by cloning. defines max versions to keep.\n\n"; + oss << "versions |fid:|fxid: [grab-version]\n" + << " List versions of a file, or grab a version [grab-version].\n\n"; + oss << "workflow |fid:|fxid: \n" + << " Trigger workflow with event on .\n"; + return oss.str(); +} + +void ConfigureFileApp(CLI::App& app, std::string& subcmd) +{ + app.name("file"); + app.description("File Handling"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeFileHelp(); + })); + app.add_option("subcmd", subcmd, + "rename|rename_with_symlink|symlink|drop|touch|move|copy|" + "replicate|purge|version|versions|layout|tag|convert|verify|" + "adjustreplica|check|share|workflow|info") + ->required(); +} + +void +AppendEncodedPath(XrdOucString& in, const XrdOucString& raw, bool absolutize) +{ + XrdOucString raw_copy = raw; + if (raw_copy.beginswith("fid:") || raw_copy.beginswith("fxid:") || + raw_copy.beginswith("pid:") || raw_copy.beginswith("pxid:") || + raw_copy.beginswith("inode:") || raw_copy.beginswith("cid:") || + raw_copy.beginswith("cxid:")) { + in += "&mgm.path="; + in += raw; + return; + } + XrdOucString path = raw; + if (absolutize) { + path = abspath(path.c_str()); + } + XrdOucString esc = + eos::common::StringConversion::curl_escaped(path.c_str()).c_str(); + in += "&mgm.path="; + in += esc; + in += "&eos.encodepath=1"; +} + +int +GetRemoteFmdFromLocalDb(const char* manager, const char* shexfid, + const char* sfsid, eos::common::FmdHelper& fmd) +{ + if ((!manager) || (!shexfid) || (!sfsid)) { + return EINVAL; + } + + int rc = 0; + XrdCl::Buffer arg; + XrdCl::Buffer* response = 0; + XrdCl::XRootDStatus status; + XrdOucString fmdquery = "/?fst.pcmd=getfmd&fst.getfmd.fid="; + fmdquery += shexfid; + fmdquery += "&fst.getfmd.fsid="; + fmdquery += sfsid; + XrdOucString address = "root://"; + address += manager; + address += "//dummy"; + XrdCl::URL url(address.c_str()); + + if (!url.IsValid()) { + eos_static_err("error=URL is not valid: %s", address.c_str()); + return EINVAL; + } + + std::unique_ptr fs(new XrdCl::FileSystem(url)); + + if (!fs) { + eos_static_err("error=failed to get new FS object"); + return EINVAL; + } + + arg.FromString(fmdquery.c_str()); + status = fs->Query(XrdCl::QueryCode::OpaqueFile, arg, response); + + if (status.IsOK()) { + rc = 0; + eos_static_debug( + "got replica file meta data from server %s for fxid=%s fsid=%s", + manager, shexfid, sfsid); + } else { + rc = ECOMM; + eos_static_err( + "Unable to retrieve meta data from server %s for fxid=%s fsid=%s", + manager, shexfid, sfsid); + } + + if (rc) { + delete response; + return EIO; + } + + if (!strncmp(response->GetBuffer(), "ERROR", 5)) { + // remote side couldn't get the record + eos_static_info( + "Unable to retrieve meta data on remote server %s for fxid=%s fsid=%s", + manager, shexfid, sfsid); + delete response; + return ENODATA; + } + + // get the remote file meta data into an env hash + XrdOucEnv fmdenv(response->GetBuffer()); + + if (!eos::common::EnvToFstFmd(fmdenv, fmd)) { + int envlen; + eos_static_err("Failed to unparse file meta data %s", fmdenv.Env(envlen)); + delete response; + return EIO; + } + + // very simple check + if (fmd.mProtoFmd.fid() != eos::common::FileId::Hex2Fid(shexfid)) { + eos_static_err("Uups! Received wrong meta data from remote server - fid " + "is %lu instead of %lu !", + fmd.mProtoFmd.fid(), + eos::common::FileId::Hex2Fid(shexfid)); + delete response; + return EIO; + } + + delete response; + return 0; +} + +int +RunFileCheck(XrdOucString path, const std::string& option, CommandContext& ctx) +{ + XrdOucString in = "mgm.cmd=file"; + + bool absolutize = (!path.beginswith("fid:")) && (!path.beginswith("fxid:")); + + in += "&mgm.subcmd=getmdlocation"; + in += "&mgm.format=fuse"; + AppendEncodedPath(in, path, absolutize); + + // Eventually disable json format to avoid parsing issues + bool old_json = json; + if (old_json) { + json = false; + } + + XrdOucEnv* result = ctx.clientCommand(in, false, nullptr); + + if (old_json) { + json = true; + } + + if (!result) { + fprintf(stderr, "error: getmdlocation query failed\n"); + global_retc = EINVAL; + return 0; + } + + int envlen = 0; + std::unique_ptr newresult(new XrdOucEnv(result->Env(envlen))); + delete result; + + if (!envlen) { + fprintf(stderr, "error: couldn't get meta data information\n"); + global_retc = EIO; + return 0; + } + + char* ptr = newresult->Get("mgm.proc.retc"); + + if (ptr) { + int retc_getmdloc = 0; + + try { + retc_getmdloc = std::stoi(ptr); + } catch (...) { + retc_getmdloc = EINVAL; + } + + if (retc_getmdloc) { + fprintf(stderr, "error: failed getmdlocation command, errno=%i", + retc_getmdloc); + global_retc = retc_getmdloc; + return 0; + } + } + + XrdOucString ns_path = newresult->Get("mgm.nspath"); + XrdOucString checksumtype = newresult->Get("mgm.checksumtype"); + XrdOucString checksum = newresult->Get("mgm.checksum"); + uint64_t mgm_size = std::stoull(newresult->Get("mgm.size")); + bool silent_cmd = ((option.find("%silent") != std::string::npos) || + ctx.silent); + + if (!silent_cmd) { + fprintf(stdout, "path=\"%s\" fxid=\"%4s\" size=\"%llu\" nrep=\"%s\" " + "checksumtype=\"%s\" checksum=\"%s\"\n", + ns_path.c_str(), newresult->Get("mgm.fid0"), + (unsigned long long)mgm_size, newresult->Get("mgm.nrep"), + checksumtype.c_str(), newresult->Get("mgm.checksum")); + } + + std::string err_label; + std::set set_errors; + int nrep_online = 0; + int i = 0; + + for (i = 0; i < 255; ++i) { + err_label = "none"; + XrdOucString repurl = "mgm.replica.url"; + repurl += i; + XrdOucString repfid = "mgm.fid"; + repfid += i; + XrdOucString repfsid = "mgm.fsid"; + repfsid += i; + XrdOucString repbootstat = "mgm.fsbootstat"; + repbootstat += i; + XrdOucString repfstpath = "mgm.fstpath"; + repfstpath += i; + + if (!newresult->Get(repurl.c_str())) { + break; + } + + // Query the FSTs for stripe info + XrdCl::StatInfo* stat_info = 0; + XrdCl::XRootDStatus status; + std::ostringstream oss; + oss << "root://" << newresult->Get(repurl.c_str()) << "//dummy"; + XrdCl::URL url(oss.str()); + + if (!url.IsValid()) { + fprintf(stderr, "error: URL is not valid: %s", oss.str().c_str()); + global_retc = EINVAL; + return 0; + } + + // Get XrdCl::FileSystem object + std::unique_ptr fs {new XrdCl::FileSystem(url)}; + + if (!fs) { + fprintf(stderr, "error: failed to get new FS object"); + global_retc = ECOMM; + return 0; + } + + XrdOucString bs = newresult->Get(repbootstat.c_str()); + bool down = (bs != "booted"); + + if (down && (option.find("%force") == std::string::npos)) { + err_label = "DOWN"; + set_errors.insert(err_label); + + if (!silent_cmd) { + fprintf(stderr, + "error: unable to retrieve file meta data from %s " + "[ status=%s ]\n", + newresult->Get(repurl.c_str()), bs.c_str()); + } + + continue; + } + + // Do a remote stat using XrdCl::FileSystem + uint64_t stat_size = std::numeric_limits::max(); + XrdOucString statpath = newresult->Get(repfstpath.c_str()); + + if (!statpath.beginswith("/")) { + // base 64 encode this path + XrdOucString statpath64; + eos::common::SymKey::Base64(statpath, statpath64); + statpath = "/#/"; + statpath += statpath64; + } + + status = fs->Stat(statpath.c_str(), stat_info); + + if (!status.IsOK()) { + err_label = "STATFAILED"; + set_errors.insert(err_label); + } else { + stat_size = stat_info->GetSize(); + } + + // Free memory + delete stat_info; + int retc = 0; + eos::common::FmdHelper fmd; + + if ((retc = GetRemoteFmdFromLocalDb(newresult->Get(repurl.c_str()), + newresult->Get(repfid.c_str()), + newresult->Get(repfsid.c_str()), + fmd))) { + if (!silent_cmd) { + fprintf(stderr, "error: unable to retrieve file meta data from %s [%d]\n", + newresult->Get(repurl.c_str()), retc); + } + + err_label = "NOFMD"; + set_errors.insert(err_label); + } else { + const auto& proto_fmd = fmd.mProtoFmd; + XrdOucString cx = proto_fmd.checksum().c_str(); + + for (unsigned int k = (cx.length() / 2); k < SHA256_DIGEST_LENGTH; ++k) { + cx += "00"; + } + + std::string disk_cx = proto_fmd.diskchecksum().c_str(); + + for (unsigned int k = (disk_cx.length() / 2); k < SHA256_DIGEST_LENGTH; + ++k) { + disk_cx += "00"; + } + + if (eos::common::LayoutId::IsRain(proto_fmd.lid()) == false) { + // These checks make sense only for non-rain layouts + if (proto_fmd.size() != mgm_size) { + err_label = "SIZE"; + set_errors.insert(err_label); + } else { + if (proto_fmd.size() != (unsigned long long) stat_size) { + err_label = "FSTSIZE"; + set_errors.insert(err_label); + } + } + + if (cx != checksum) { + err_label = "CHECKSUM"; + set_errors.insert(err_label); + } + + uint64_t disk_cx_val = 0ull; + + try { + disk_cx_val = std::stoull(disk_cx.substr(0, 8), nullptr, 16); + } catch (...) { + // error during conversion + } + + if ((disk_cx.length() > 0) && disk_cx_val && + ((disk_cx.length() < 8) || (!cx.beginswith(disk_cx.c_str())))) { + err_label = "DISK_CHECKSUM"; + set_errors.insert(err_label); + } + + if (!silent_cmd) { + fprintf(stdout, + "nrep=\"%02d\" fsid=\"%s\" host=\"%s\" fstpath=\"%s\" " + "size=\"%llu\" statsize=\"%llu\" checksum=\"%s\" " + "diskchecksum=\"%s\" error_label=\"%s\"\n", + i, newresult->Get(repfsid.c_str()), + newresult->Get(repurl.c_str()), + newresult->Get(repfstpath.c_str()), + (unsigned long long)proto_fmd.size(), + (unsigned long long)(stat_size), + cx.c_str(), disk_cx.c_str(), err_label.c_str()); + } + } else { + // For RAIN layouts we only check for block-checksum errors + if (proto_fmd.blockcxerror()) { + err_label = "BLOCK_XS"; + set_errors.insert(err_label); + } + + if (!silent_cmd) { + fprintf(stdout, + "nrep=\"%02d\" fsid=\"%s\" host=\"%s\" fstpath=\"%s\" " + "size=\"%llu\" statsize=\"%llu\" error_label=\"%s\"\n", + i, newresult->Get(repfsid.c_str()), + newresult->Get(repurl.c_str()), + newresult->Get(repfstpath.c_str()), + (unsigned long long)proto_fmd.size(), + (unsigned long long)(stat_size), err_label.c_str()); + } + } + + ++nrep_online; + } + } + + int nrep = 0; + int stripes = 0; + + if (newresult->Get("mgm.stripes")) { + stripes = atoi(newresult->Get("mgm.stripes")); + } + + if (newresult->Get("mgm.nrep")) { + nrep = atoi(newresult->Get("mgm.nrep")); + } + + if (nrep != stripes) { + if (set_errors.find("NOFMD") == set_errors.end()) { + err_label = "NUM_REPLICAS"; + set_errors.insert(err_label); + } + } + + if (set_errors.size()) { + if ((option.find("%output")) != std::string::npos) { + fprintf(stdout, "INCONSISTENCY %s path=%-32s fxid=%s size=%llu " + "stripes=%d nrep=%d nrepstored=%d nreponline=%d " + "checksumtype=%s checksum=%s\n", set_errors.begin()->c_str(), + path.c_str(), newresult->Get("mgm.fid0"), + (unsigned long long) mgm_size, stripes, nrep, i, nrep_online, + checksumtype.c_str(), newresult->Get("mgm.checksum")); + } + + if (((option.find("%size") != std::string::npos) && + ((set_errors.find("SIZE") != set_errors.end() || + set_errors.find("FSTSIZE") != set_errors.end()))) || + ((option.find("%checksum") != std::string::npos) && + ((set_errors.find("CHECKSUM") != set_errors.end()) || + (set_errors.find("BLOCK_XS") != set_errors.end()))) || + ((option.find("%diskchecksum") != std::string::npos) && + (set_errors.find("DISK_CHECKSUM") != set_errors.end())) || + ((option.find("%nrep") != std::string::npos) && + ((set_errors.find("NOFMD") != set_errors.end()) || + (set_errors.find("NUM_REPLICAS") != set_errors.end())))) { + global_retc = EFAULT; + } + } + + return 0; +} + +class FileCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "file"; + } + const char* + description() const override + { + return "File Handling"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + std::string subcmd; + ConfigureFileApp(app, subcmd); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::vector remaining = app.remaining(); + + XrdOucString cmd = subcmd.c_str(); + std::vector rest = remaining; + XrdOucString in = "mgm.cmd=file"; + + auto set_path_or_id = [&](XrdOucString path) { + if (Path2FileDenominator(path)) { + in += "&mgm.file.id="; + in += path; + } else { + AppendEncodedPath(in, path, true); + } + }; + + auto is_path_or_id = [](const std::string& s) -> bool { + return !s.empty() && (s[0] == '/' || s.find("fid:") == 0 || + s.find("fxid:") == 0 || s.find("pid:") == 0 || + s.find("pxid:") == 0 || s.find("inode:") == 0 || + s.find("cid:") == 0 || s.find("cxid:") == 0 || + s[0] != '-'); + }; + + if (cmd == "rename") { + if (rest.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.subcmd=rename"; + XrdOucString p = abspath(rest[0].c_str()); + set_path_or_id(p); + in += "&mgm.file.source="; + in += p; + in += "&mgm.file.target="; + in += abspath(rest[1].c_str()); + } else if (cmd == "rename_with_symlink") { + if (rest.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.subcmd=rename_with_symlink"; + XrdOucString p = abspath(rest[0].c_str()); + set_path_or_id(p); + in += "&mgm.file.source="; + in += p; + in += "&mgm.file.target="; + in += abspath(rest[1].c_str()); + } else if (cmd == "symlink") { + std::vector positionals; + bool force = false; + + for (const auto& arg : rest) { + if (arg == "-f") + force = true; + else + positionals.push_back(arg); + } + if (positionals.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.subcmd=symlink"; + // positionals[0]=link (symlink name), positionals[1]=target (after CLI11 reverse) + XrdOucString p = abspath(positionals[0].c_str()); + set_path_or_id(p); + in += "&mgm.file.source="; + in += p; + in += "&mgm.file.target="; + in += positionals[1].c_str(); + if (force) + in += "&mgm.file.force=1"; + } else if (cmd == "drop") { + if (rest.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=drop"; + set_path_or_id(p); + in += "&mgm.file.fsid="; + in += rest[1].c_str(); + if (rest.size() > 2 && rest[2] == "-f") + in += "&mgm.file.force=1"; + } else if (cmd == "touch") { + std::string option; + size_t idx = 0; + for (; idx < rest.size(); ++idx) { + if (!rest[idx].empty() && rest[idx][0] == '-') { + std::string tmp = rest[idx]; + tmp.erase(std::remove(tmp.begin(), tmp.end(), '-'), tmp.end()); + option += tmp; + } else { + break; + } + } + if (idx >= rest.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[idx].c_str()); + in += "&mgm.subcmd=touch"; + set_path_or_id(p); + std::string fsid1 = (idx + 1 < rest.size()) ? rest[idx + 1] : ""; + std::string fsid2 = (idx + 2 < rest.size()) ? rest[idx + 2] : ""; + + if (option.find('n') != std::string::npos) { + in += "&mgm.file.touch.nolayout=true"; + } + if (option.find('0') != std::string::npos) { + in += "&mgm.file.touch.truncate=true"; + } + if (option.find('a') != std::string::npos) { + in += "&mgm.file.touch.absorb=true"; + } + if (option.find('l') != std::string::npos) { + in += "&mgm.file.touch.lockop=lock"; + if (!fsid1.empty()) { + in += "&mgm.file.touch.lockop.lifetime="; + in += fsid1.c_str(); + fsid1.clear(); + } + if (!fsid2.empty()) { + if ((fsid2 != "app") && (fsid2 != "user")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (fsid2 == "app") { + // this is inverted logic because we set the wildcard + in += "&mgm.file.touch.wildcard=user"; + } else { + // this is inverted logic because we set the wildcard + in += "&mgm.file.touch.wildcard=app"; + } + fsid2.clear(); + } + } + if (option.find('u') != std::string::npos) { + in += "&mgm.file.touch.lockop=unlock"; + fsid1.clear(); + fsid2.clear(); + } + if (!fsid1.empty()) { + if (!fsid1.empty() && fsid1[0] == '/') { + in += "&mgm.file.touch.hardlinkpath="; + in += fsid1.c_str(); + } else { + in += "&mgm.file.touch.size="; + in += fsid1.c_str(); + } + } + if (!fsid2.empty()) { + in += "&mgm.file.touch.checksuminfo="; + in += fsid2.c_str(); + } + } else if (cmd == "move") { + if (rest.size() < 3) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=move"; + set_path_or_id(p); + in += "&mgm.file.sourcefsid="; + in += rest[1].c_str(); + in += "&mgm.file.targetfsid="; + in += rest[2].c_str(); + } else if (cmd == "copy") { + std::string option; + size_t idx = 0; + for (; idx < rest.size(); ++idx) { + if (!rest[idx].empty() && rest[idx][0] == '-') { + std::string tmp = rest[idx]; + tmp.erase(std::remove(tmp.begin(), tmp.end(), '-'), tmp.end()); + option += tmp; + } else { + break; + } + } + if (idx + 1 >= rest.size()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[idx].c_str()); + XrdOucString dest = rest[idx + 1].c_str(); + in += "&mgm.subcmd=copy"; + set_path_or_id(p); + if (!option.empty()) { + std::string checkoption = option; + checkoption.erase( + std::remove(checkoption.begin(), checkoption.end(), 'f'), + checkoption.end()); + checkoption.erase( + std::remove(checkoption.begin(), checkoption.end(), 's'), + checkoption.end()); + checkoption.erase( + std::remove(checkoption.begin(), checkoption.end(), 'c'), + checkoption.end()); + if (!checkoption.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.file.option="; + in += option.c_str(); + } + dest = abspath(dest.c_str()); + in += "&mgm.file.target="; + in += dest; + } else if (cmd == "replicate") { + if (rest.size() < 3) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=replicate"; + set_path_or_id(p); + in += "&mgm.file.sourcefsid="; + in += rest[1].c_str(); + in += "&mgm.file.targetfsid="; + in += rest[2].c_str(); + } else if (cmd == "purge" || cmd == "version") { + if (rest.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.subcmd="; + in += cmd; + XrdOucString p = abspath(rest[0].c_str()); + AppendEncodedPath(in, p, true); + in += "&mgm.purge.version="; + if (rest.size() > 1) + in += rest[1].c_str(); + else + in += "-1"; + } else if (cmd == "versions") { + if (rest.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=versions"; + set_path_or_id(p); + in += "&mgm.grab.version="; + in += (rest.size() > 1 ? rest[1].c_str() : "-1"); + } else if (cmd == "layout") { + if (rest.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=layout"; + set_path_or_id(p); + if (rest[1] == "-stripes" && rest.size() > 2) { + in += "&mgm.file.layout.stripes="; + in += rest[2].c_str(); + } else if (rest[1] == "-checksum" && rest.size() > 2) { + in += "&mgm.file.layout.checksum="; + in += rest[2].c_str(); + } else if (rest[1] == "-type" && rest.size() > 2) { + in += "&mgm.file.layout.type="; + in += rest[2].c_str(); + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } else if (cmd == "tag") { + if (rest.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=tag"; + set_path_or_id(p); + in += "&mgm.file.tag.fsid="; + in += rest[1].c_str(); + } else if (cmd == "convert") { + bool rewrite = false; + std::vector positionals; + + for (const auto& arg : rest) { + if (arg == "--rewrite") + rewrite = true; + else if (arg == "--sync") { + fprintf(stderr, "error: --sync is currently not supported\n"); + printHelp(); + global_retc = EINVAL; + return 0; + } else + positionals.push_back(arg); + } + if (positionals.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(positionals[0].c_str()); + in += "&mgm.subcmd=convert"; + set_path_or_id(p); + if (positionals.size() > 1) { + in += "&mgm.convert.layout="; + in += positionals[1].c_str(); + } + if (positionals.size() > 2) { + in += "&mgm.convert.space="; + in += positionals[2].c_str(); + } + if (positionals.size() > 3) { + in += "&mgm.convert.placementpolicy="; + in += positionals[3].c_str(); + } + if (positionals.size() > 4) { + in += "&mgm.convert.checksum="; + in += positionals[4].c_str(); + } + if (rewrite) + in += "&mgm.option=rewrite"; + } else if (cmd == "verify") { + std::string path; + std::string filter_fsid; + std::string rate_val; + + for (size_t i = 0; i < rest.size(); ++i) { + const std::string& opt = rest[i]; + if (opt == "-checksum") + in += "&mgm.file.compute.checksum=1"; + else if (opt == "-commitchecksum") + in += "&mgm.file.commit.checksum=1"; + else if (opt == "-commitsize") + in += "&mgm.file.commit.size=1"; + else if (opt == "-commitfmd") + in += "&mgm.file.commit.fmd=1"; + else if (opt == "-rate") { + if (i + 1 < rest.size()) { + rate_val = rest[++i]; + in += "&mgm.file.verify.rate="; + in += rate_val.c_str(); + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } else if (opt == "-resync") + in += "&mgm.file.resync=1"; + else if (!path.empty() && !opt.empty() && + std::isdigit(static_cast(opt[0]))) { + filter_fsid = opt; + in += "&mgm.file.verify.filterid="; + in += filter_fsid.c_str(); + } else if (is_path_or_id(opt)) { + if (path.empty()) + path = opt; + else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + if (path.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(path.c_str()); + in += "&mgm.subcmd=verify"; + AppendEncodedPath(in, p, true); + } else if (cmd == "adjustreplica") { + if (rest.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=adjustreplica"; + set_path_or_id(p); + std::vector args(rest.begin() + 1, rest.end()); + int positional_index = 0; + for (size_t i = 0; i < args.size(); ++i) { + if (args[i] == "--exclude-fs") { + if (i + 1 < args.size()) { + in += "&mgm.file.excludefs="; + in += args[i + 1].c_str(); + i++; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } else if (args[i] == "--nodrop") { + in += "&mgm.file.nodrop=1"; + } else { + if (positional_index == 0) { + in += "&mgm.file.desiredspace="; + in += args[i].c_str(); + positional_index++; + } else if (positional_index == 1) { + in += "&mgm.file.desiredsubgroup="; + in += args[i].c_str(); + positional_index++; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + } + } else if (cmd == "check") { + if (rest.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + std::string option = (rest.size() > 1) ? rest[1] : ""; + return RunFileCheck(rest[0].c_str(), option, ctx); + } else if (cmd == "share") { + if (rest.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=share"; + AppendEncodedPath(in, p, true); + unsigned long long expires = (28ull * 86400ull); + if (rest.size() > 1) { + in += "&mgm.file.expires="; + in += rest[1].c_str(); + } else { + char buf[64]; + snprintf(buf, sizeof(buf), "%llu", expires); + in += "&mgm.file.expires="; + in += buf; + } + } else if (cmd == "workflow") { + if (rest.size() < 3) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString p = abspath(rest[0].c_str()); + in += "&mgm.subcmd=workflow"; + AppendEncodedPath(in, p, true); + in += "&mgm.workflow="; + in += rest[1].c_str(); + in += "&mgm.event="; + in += rest[2].c_str(); + } else if (cmd == "info") { + std::string path; + XrdOucString option = ""; + + for (const auto& arg : rest) { + if (is_path_or_id(arg)) { + if (path.empty()) + path = arg; + else { + printHelp(); + global_retc = EINVAL; + return 0; + } + } else { + XrdOucString tok = arg.c_str(); + if (tok == "s" || tok == "-s" || tok == "--silent") + option += "silent"; + else + option += tok; + } + } + if (path.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString path_str = path.c_str(); + bool absolutize = (!path_str.beginswith("fid:")) && (!path_str.beginswith("fxid:")) && + (!path_str.beginswith("pid:")) && (!path_str.beginswith("pxid:")) && + (!path_str.beginswith("inode:")); + XrdOucString fin = "mgm.cmd=fileinfo"; + AppendEncodedPath(fin, path_str, absolutize); + if (option.length()) { + fin += "&mgm.file.info.option="; + fin += option; + } + if (option.find("silent") == STR_NPOS) { + global_retc = + ctx.outputResult(ctx.clientCommand(fin, false, nullptr), true); + } + return 0; + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeFileHelp().c_str()); + } +}; +} // namespace + +void +RegisterFileNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/fileinfo-alias.cc b/console/commands/native/fileinfo-alias.cc new file mode 100644 index 0000000000..c9ff66d5d0 --- /dev/null +++ b/console/commands/native/fileinfo-alias.cc @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------- +// File: fileinfo-alias.cc +// Purpose: Provide 'fileinfo' alias for 'file info ...' +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include + +namespace { +class FileInfoAliasCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "fileinfo"; + } + const char* + description() const override + { + return "Alias for 'file info'"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + IConsoleCommand* fileCmd = CommandRegistry::instance().find("file"); + if (!fileCmd) { + fprintf(stderr, "error: 'file' command not available\n"); + return -1; + } + std::vector forwarded; + forwarded.reserve(args.size() + 1); + forwarded.emplace_back("info"); + forwarded.insert(forwarded.end(), args.begin(), args.end()); + return fileCmd->run(forwarded, ctx); + } + void + printHelp() const override + { + fprintf(stderr, + "Usage: fileinfo [options] (alias for 'file info')\n"); + } +}; +} // namespace + +void +RegisterFileInfoAliasCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/find-proto-native.cc b/console/commands/native/find-proto-native.cc new file mode 100644 index 0000000000..06f3815eed --- /dev/null +++ b/console/commands/native/find-proto-native.cc @@ -0,0 +1,154 @@ +// ---------------------------------------------------------------------- +// File: find-proto-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include "console/commands/helpers/NewfindHelper.hh" +#include +#include +#include + +namespace { +std::string MakeFindHelp() +{ + return "Usage: find [OPTIONS] \n\n" + "Find files and directories. OPTIONS can be filters, actions, or output modifiers.\n\n" + "Filters: [--maxdepth ] [--name ] [-f] [-d] [-0] [-g] " + "[-uid ] [-nuid ] [-gid ] [-ngid ] [-flag ] [-nflag ] " + "[--ctime|--mtime +|-] [-x =] [--faultyacl] [--stripediff]\n\n" + "Actions: [-b] [--layoutstripes ] [--purge ] [--fileinfo] " + "[--format formatlist] [--cache] [--du]\n\n" + "Output: [--xurl] [-p ] [--nrep] [--nunlink] [--size] [--online] " + "[--hosts] [--partition] [--fid] [--fs] [--checksum] [--ctime] [--mtime] " + "[--uid] [--gid]\n\n" + " can be: file:... (local), root:... (XRootD), as3:... (S3), or EOS path.\n"; +} + +void ConfigureFindApp(CLI::App& app) +{ + app.name("find"); + app.description("Find files/directories"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeFindHelp(); + })); +} + +class FindProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "find"; + } + const char* + description() const override + { + return "Find files/directories"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + // Reuse the same helper as the original newfind implementation + NewfindHelper finder(*ctx.globalOpts); + // Special schemes handled locally + if (joined.find("root://") != std::string::npos) { + std::string path = joined.substr(joined.rfind("root://")); + path.erase(std::remove(path.begin(), path.end(), '"'), path.end()); + global_retc = finder.FindXroot(path); + return 0; + } else if (joined.find("file:") != std::string::npos) { + std::string path = joined.substr(joined.rfind("file:")); + path.erase(std::remove(path.begin(), path.end(), '"'), path.end()); + global_retc = finder.FindXroot(path); + return 0; + } else if (joined.find("as3:") != std::string::npos) { + std::string path = joined.substr(joined.rfind("as3:")); + path.erase(std::remove(path.begin(), path.end(), '"'), path.end()); + global_retc = finder.FindAs3(path); + return 0; + } + if (!finder.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = finder.Execute(); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureFindApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; + +// Provide 'newfind' alias to the same implementation as 'find' +class NewfindAliasCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "newfind"; + } + const char* + description() const override + { + return "Find files/directories (new)"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + IConsoleCommand* findCmd = CommandRegistry::instance().find("find"); + if (!findCmd) { + fprintf(stderr, "error: 'find' command not available\n"); + global_retc = EINVAL; + return 0; + } + return findCmd->run(args, ctx); + } + void + printHelp() const override + { + // Delegate to 'find' help + IConsoleCommand* findCmd = CommandRegistry::instance().find("find"); + if (findCmd) + findCmd->printHelp(); + } +}; +} // namespace + +void +RegisterFindProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/fs-proto-native.cc b/console/commands/native/fs-proto-native.cc new file mode 100644 index 0000000000..abd5e35a0d --- /dev/null +++ b/console/commands/native/fs-proto-native.cc @@ -0,0 +1,107 @@ +// ---------------------------------------------------------------------- +// File: fs-proto-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include "console/commands/helpers/FsHelper.hh" +#include +#include +#include + +namespace { +std::string MakeFsHelp() +{ + return "Usage: fs add|boot|clone|compare|config|dropdeletion|dropghosts|" + "dropfiles|dumpmd|ls|mv|rm|status [OPTIONS]\n\n" + " fs add [-m|--manual ] |[:] " + " [ [ []]]\n" + " fs boot |||* [--syncdisk|--syncmgm]\n" + " fs clone \n" + " fs compare \n" + " fs config =\n" + " fs dropdeletion \n" + " fs dropghosts [--fxid fid1 [fid2] ...]\n" + " fs dropfiles [-f]\n" + " fs dumpmd [--count] [--fid|--fxid|--path] [--size] [-m|-s]\n" + " fs ls [-m|-l|-e|--io|--fsck|[-d|--drain]|-D|-F] [-s] [-b|--brief] " + "[[matchlist]]\n" + " fs mv [--force] \n" + " fs rm || | \n" + " fs status [-r] [-l] \n"; +} + +void ConfigureFsApp(CLI::App& app) +{ + app.name("fs"); + app.description("File System configuration"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeFsHelp(); + })); +} + +class FsProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "fs"; + } + const char* + description() const override + { + return "File System configuration"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + // Quote args with spaces so FsHelper's tokenizer preserves them + if (args[i].find(' ') != std::string::npos) + oss << std::quoted(args[i]); + else + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + FsHelper helper(*ctx.globalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + // Confirmation is handled inside FsHelper where applicable + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureFsApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterFsProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/fsck-proto-native.cc b/console/commands/native/fsck-proto-native.cc new file mode 100644 index 0000000000..6d5fd8d883 --- /dev/null +++ b/console/commands/native/fsck-proto-native.cc @@ -0,0 +1,125 @@ +// ---------------------------------------------------------------------- +// File: fsck-proto-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/FsckHelper.hh" +#include +#include +#include + +namespace { +std::string MakeFsckHelp() +{ + std::ostringstream oss; + oss << "Usage: fsck [stat|config|report|repair|clean_orphans]\n" + << " control and display file system check information\n" + << std::endl + << " fsck stat [-m]\n" + << " print summary of consistency checks\n" + << " -m : print in monitoring format\n" + << std::endl + << " fsck config \n" + << " configure the fsck with the following possible options:\n" + << " collect-interval-min : collection interval in minutes [default 30]\n" + << " collect : control error collection thread - on/off\n" + << " repair : control error repair thread - on/off\n" + << " best-effort : control best-effort repair mode - on/off\n" + << " repair-category : specify error types that the repair thread will handle\n" + << " e.g all, m_cx_diff, m_mem_sz_diff, d_cx_diff, d_mem_sz_diff,\n" + << " unreg_n, rep_diff_n, rep_missing_n, blockxs_err, stripe_err\n" + << " max-queued-jobs : maximum number of queued jobs\n" + << " max-thread-pool-size : maximum number of threads in the fsck pool\n" + << " show-dark-files : on/off [default off] - might affect instance performance\n" + << " show-offline : on/off [default off] - might affect instance performance\n" + << " show-no-replica : on/off [default off] - might affect instance performance\n" + << std::endl + << " fsck report [-a] [-i] [-l] [-j|--json] [--error ...]\n" + << " report consistency check results, with the following options\n" + << " -a : break down statistics per file system\n" + << " -i : display file identifiers\n" + << " -l : display logical file name\n" + << " -j|--json : display in JSON output format\n" + << " --error : display information about the following error tags\n" + << std::endl + << " fsck repair --fxid [--fsid ] [--error ] [--async]\n" + << " repair the given file if there are any errors\n" + << " --fxid : hexadecimal file identifier\n" + << " --fsid : file system id used for collecting info\n" + << " --error : error type for given file system id e.g. m_cx_diff unreg_n etc\n" + << " --async : job queued and ran by the repair thread if enabled\n" + << std::endl + << " fsck clean_orphans [--fsid ] [--force-qdb-cleanup]\n" + << " clean orphans by removing the entries from disk and local\n" + << " database for all file systems or only for the given fsid.\n" + << " This operation is synchronous but the fsck output will be\n" + << " updated once the inconsistencies are refreshed.\n" + << " --force-qdb-cleanup : force remove orphan entries from qdb\n" + << std::endl; + return oss.str(); +} + +class FsckProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "fsck"; + } + const char* + description() const override + { + return "File System Consistency Checking"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + // Quote args with spaces so FsckHelper's tokenizer preserves them + if (args[i].find(' ') != std::string::npos) + oss << std::quoted(args[i]); + else + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (args.empty()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + FsckHelper helper(*ctx.globalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeFsckHelp().c_str()); + } +}; +} // namespace + +void +RegisterFsckProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/fuse-native.cc b/console/commands/native/fuse-native.cc new file mode 100644 index 0000000000..3ab9074ab4 --- /dev/null +++ b/console/commands/native/fuse-native.cc @@ -0,0 +1,142 @@ +// ---------------------------------------------------------------------- +// File: fuse-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include + +namespace { +class FuseCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "fuse"; + } + const char* + description() const override + { + return "Fuse Mounting"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + if (interactive) { + fprintf(stderr, "error: don't call from an interactive shell - " + "call via 'eos fuse ...'!\n"); + global_retc = -1; + return 0; + } + if (!args.empty()) { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + if (wants_help(oss.str().c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + if (args.empty() || (args[0] != "mount" && args[0] != "umount")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString cmd = args[0].c_str(); + XrdOucString mountpoint = (args.size() > 1 ? args[1].c_str() : ""); + XrdCl::URL url(serveruri.c_str()); + XrdOucString params = "fsname="; + params += (url.GetHostName() == "localhost") ? "localhost.localdomain" + : url.GetHostName().c_str(); + params += ":"; + params += url.GetPath().c_str(); + if (cmd == "mount") { + if (!mountpoint.length()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + struct stat buf {}; + if (stat(mountpoint.c_str(), &buf)) { + XrdOucString createdir = "mkdir -p "; + createdir += mountpoint; + createdir += " >& /dev/null"; + fprintf(stderr, ".... trying to create ... %s\n", mountpoint.c_str()); + int rc = system(createdir.c_str()); + if (WEXITSTATUS(rc)) + fprintf(stderr, "error: creation of mountpoint failed"); + } + if (stat(mountpoint.c_str(), &buf)) { + fprintf(stderr, "error: cannot create mountpoint %s !\n", + mountpoint.c_str()); + return -1; + } +#ifdef __APPLE__ + params += " -onoappledouble,allow_root,defer_permissions,volname=EOS," + "iosize=65536,fsname=eos@cern.ch"; +#endif + fprintf(stderr, "===> Mountpoint : %s\n", mountpoint.c_str()); + fprintf(stderr, "===> Fuse-Options : %s\n", params.c_str()); + XrdOucString mount; + mount = "eosxd "; + mount += mountpoint.c_str(); + mount += " -o"; + mount += params; + mount += " >& /dev/null"; + int rc = system(mount.c_str()); + if (WEXITSTATUS(rc)) { + fprintf(stderr, "error: failed mount\n"); + return -1; + } + fprintf(stderr, "info: successfully mounted EOS [%s] under %s\n", + serveruri.c_str(), mountpoint.c_str()); + } else { + if (!mountpoint.length()) { + printHelp(); + global_retc = EINVAL; + return 0; + } +#ifdef __APPLE__ + XrdOucString umount = "umount -f "; + umount += mountpoint.c_str(); + umount += " >& /dev/null"; +#else + XrdOucString umount = "fusermount -z -u "; + umount += mountpoint.c_str(); +#endif + int rc = system(umount.c_str()); + if (WEXITSTATUS(rc)) + fprintf(stderr, "error: umount failed\n"); + } + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "Usage:\n" + " fuse mount \n" + " fuse umount \n" + "Mount uses server URI to derive fsname and prepares the " + "mountpoint.\n"); + } +}; +} // namespace + +void +RegisterFuseNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/fusex-cmd-native.cc b/console/commands/native/fusex-cmd-native.cc new file mode 100644 index 0000000000..867c5805bd --- /dev/null +++ b/console/commands/native/fusex-cmd-native.cc @@ -0,0 +1,194 @@ +// ---------------------------------------------------------------------- +// File: fusex-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringConversion.hh" +#include "common/SymKeys.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include +#include + +namespace { +std::string MakeFusexHelp() +{ + return "Usage: fusex [args...]\n\n" + "Subcommands:\n" + " ls list active FUSEX clients\n" + " evict [reason] evict a client by UUID " + "(reason base64-encoded)\n" + " caps [all|token|lock] [filter] show capabilities, optional " + "filter string\n" + " dropcaps drop capabilities for client " + "UUID\n" + " droplocks drop locks for inode and " + "process\n" + " conf [hb] [qc] [bc.max] [bc.match] configure heartbeat (hb), " + "queue cap (qc),\n" + " block cache max and match\n"; +} + +void ConfigureFusexApp(CLI::App& app, std::string& subcmd) +{ + app.name("fusex"); + app.description("Fuse(x) Administration"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeFusexHelp(); + })); + app.add_option("subcmd", subcmd, + "ls|evict|caps|dropcaps|droplocks|conf") + ->required(); +} + +class FusexCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "fusex"; + } + const char* + description() const override + { + return "Fuse(x) Administration"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + std::string subcmd; + ConfigureFusexApp(app, subcmd); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::vector remaining = app.remaining(); + std::reverse(remaining.begin(), remaining.end()); + + XrdOucString in = "mgm.cmd=fusex"; + if (subcmd == "ls") { + in += "&mgm.subcmd=ls"; + } else if (subcmd == "evict") { + if (remaining.size() < 1) { + printHelp(); + global_retc = EINVAL; + return 0; + } + XrdOucString uuid = remaining[0].c_str(); + in += "&mgm.subcmd=evict&mgm.fusex.uuid="; + in += uuid; + if (remaining.size() > 1) { + XrdOucString reason = remaining[1].c_str(); + XrdOucString b64; + eos::common::SymKey::Base64(reason, b64); + in += "&mgm.fusex.reason="; + in += b64; + } + } else if (subcmd == "caps") { + XrdOucString option = (remaining.size() > 0 ? remaining[0].c_str() : ""); + option.replace("-", ""); + in += "&mgm.subcmd=caps&mgm.option="; + in += option; + if (remaining.size() > 1) { + std::ostringstream f; + for (size_t i = 1; i < remaining.size(); ++i) { + if (i > 1) + f << ' '; + f << remaining[i]; + } + XrdOucString filter = f.str().c_str(); + if (filter.length()) { + in += "&mgm.filter="; + in += eos::common::StringConversion::curl_escaped(filter.c_str()) + .c_str(); + } + } + } else if (subcmd == "dropcaps") { + if (remaining.size() < 1) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.subcmd=dropcaps&mgm.fusex.uuid="; + in += remaining[0].c_str(); + } else if (subcmd == "droplocks") { + if (remaining.size() < 2) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in += "&mgm.subcmd=droplocks&mgm.inode="; + in += remaining[0].c_str(); + in += "&mgm.fusex.pid="; + in += remaining[1].c_str(); + } else if (subcmd == "conf") { + in += "&mgm.subcmd=conf"; + if (remaining.size() > 0) { + in += "&mgm.fusex.hb="; + in += remaining[0].c_str(); + } + if (remaining.size() > 1) { + in += "&mgm.fusex.qc="; + in += remaining[1].c_str(); + } + if (remaining.size() > 2) { + in += "&mgm.fusex.bc.max="; + in += remaining[2].c_str(); + } + if (remaining.size() > 3) { + in += "&mgm.fusex.bc.match="; + in += remaining[3].c_str(); + } + } else { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = ctx.outputResult(ctx.clientCommand(in, true, nullptr), true); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeFusexHelp().c_str()); + } +}; +} // namespace + +void +RegisterFusexNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/geosched-cmd-native.cc b/console/commands/native/geosched-cmd-native.cc new file mode 100644 index 0000000000..4a6cb05377 --- /dev/null +++ b/console/commands/native/geosched-cmd-native.cc @@ -0,0 +1,357 @@ +// ---------------------------------------------------------------------- +// File: geosched-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "common/Utils.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include +#include + +namespace { +std::string MakeGeoschedHelp() +{ + return "Usage: geosched show|set|updater|forcerefresh|disabled|access ...\n\n" + "'[eos] geosched ..' Interact with the file geoscheduling engine in " + "EOS.\n\n" + "Subcommands:\n" + " show [-c|-m] tree [] show scheduling trees\n" + " show [-c|-m] snapshot [] [] show snapshots\n" + " show param show internal parameters\n" + " show state [-m] show internal state\n" + " set [index] set parameter value\n" + " updater pause|resume pause/resume tree updater\n" + " forcerefresh force refresh\n" + " disabled add|rm|show \n" + " access setdirect|showdirect|cleardirect|setproxygroup|showproxygroup|clearproxygroup ...\n\n" + "Options:\n" + " -c enable color display\n" + " -m list in monitoring format\n\n" + "Note: Geotags must be alphanumeric segments, max 8 chars, format " + "::::...::\n"; +} + +void ConfigureGeoschedApp(CLI::App& app, std::string& subcmd) +{ + app.name("geosched"); + app.description("Geographical scheduler control"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeGeoschedHelp(); + })); + app.add_option("subcmd", subcmd, + "show|set|updater|forcerefresh|disabled|access") + ->required(); +} + +class GeoschedCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "geosched"; + } + const char* + description() const override + { + return "Geographical scheduler control"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.empty() || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + std::string subcmd; + ConfigureGeoschedApp(app, subcmd); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + eos::common::StringTokenizer subtokenizer(joined.c_str()); + subtokenizer.GetLine(); + XrdOucString cmd = subtokenizer.GetToken(); + std::set supportedParam = { + "skipSaturatedAccess", "skipSaturatedDrnAccess", + "skipSaturatedBlcAccess", "plctDlScorePenalty", + "plctUlScorePenalty", "accessDlScorePenalty", + "accessUlScorePenalty", "fillRatioLimit", + "fillRatioCompTol", "saturationThres", + "timeFrameDurationMs", "penaltyUpdateRate", + "proxyCloseToFs"}; + XrdOucString in = ""; + if ((cmd != "show") && (cmd != "set") && (cmd != "updater") && + (cmd != "forcerefresh") && (cmd != "disabled") && (cmd != "access")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + in = "mgm.cmd=geosched"; + if (cmd == "show") { + XrdOucString subcmd = subtokenizer.GetToken(); + if (subcmd == "-c") { + in += "&mgm.usecolors=1"; + subcmd = subtokenizer.GetToken(); + } else if (subcmd == "-m") { + in += "&mgm.monitoring=1"; + subcmd = subtokenizer.GetToken(); + } + if ((subcmd != "tree") && (subcmd != "snapshot") && (subcmd != "state") && + (subcmd != "param")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (subcmd == "state") { + in += "&mgm.subcmd=showstate"; + subcmd = subtokenizer.GetToken(); + if (subcmd == "-m") { + in += "&mgm.monitoring=1"; + } + } + if (subcmd == "param") { + in += "&mgm.subcmd=showparam"; + } + if (subcmd == "tree") { + in += "&mgm.subcmd=showtree"; + in += "&mgm.schedgroup="; + XrdOucString group = subtokenizer.GetToken(); + if (group.length()) { + in += group; + } + } + if (subcmd == "snapshot") { + in += "&mgm.subcmd=showsnapshot"; + in += "&mgm.schedgroup="; + XrdOucString group = subtokenizer.GetToken(); + if (group.length()) { + in += group; + } + in += "&mgm.optype="; + XrdOucString optype = subtokenizer.GetToken(); + if (optype.length()) { + in += optype; + } + } + } + if (cmd == "set") { + XrdOucString parameter = subtokenizer.GetToken(); + if (!parameter.length()) { + fprintf(stderr, "Error: parameter name is not provided\n"); + printHelp(); + global_retc = EINVAL; + return 0; + } + if (supportedParam.find(parameter.c_str()) == supportedParam.end()) { + fprintf(stderr, "Error: parameter %s not supported\n", + parameter.c_str()); + return 0; + } + XrdOucString index = subtokenizer.GetToken(); + XrdOucString value = subtokenizer.GetToken(); + if (!index.length()) { + fprintf(stderr, "Error: value is not provided\n"); + printHelp(); + global_retc = EINVAL; + return 0; + } + if (!value.length()) { + value = index; + index = "-1"; + } + double didx = 0.0; + if (!sscanf(value.c_str(), "%lf", &didx)) { + fprintf(stderr, + "Error: parameter %s should have a numeric value, %s was " + "provided\n", + parameter.c_str(), value.c_str()); + return 0; + } + if (!XrdOucString(index.c_str()).isdigit()) { + fprintf(stderr, + "Error: index for parameter %s should have a numeric value, %s " + "was provided\n", + parameter.c_str(), index.c_str()); + return 0; + } + in += "&mgm.subcmd=set"; + in += "&mgm.param="; + in += parameter.c_str(); + in += "&mgm.paramidx="; + in += index.c_str(); + in += "&mgm.value="; + in += value.c_str(); + } + if (cmd == "updater") { + XrdOucString subcmd = subtokenizer.GetToken(); + if (subcmd == "pause") { + in += "&mgm.subcmd=updtpause"; + } + if (subcmd == "resume") { + in += "&mgm.subcmd=updtresume"; + } + } + if (cmd == "forcerefresh") { + in += "&mgm.subcmd=forcerefresh"; + } + if (cmd == "disabled") { + XrdOucString subcmd = subtokenizer.GetToken(); + XrdOucString geotag, group, optype; + if ((subcmd != "add") && (subcmd != "rm") && (subcmd != "show")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + geotag = subtokenizer.GetToken(); + optype = subtokenizer.GetToken(); + group = subtokenizer.GetToken(); + if (!group.length() || !optype.length() || !geotag.length()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + std::string sgroup(group.c_str()), soptype(optype.c_str()), + sgeotag(geotag.c_str()); + const char fbdChars[] = "&/,;%$#@!*"; + auto fbdMatch = sgroup.find_first_of(fbdChars); + if (fbdMatch != std::string::npos && !(sgroup == "*")) { + fprintf(stderr, "illegal character %c detected in group name %s\n", + sgroup[fbdMatch], sgroup.c_str()); + return 0; + } + fbdMatch = soptype.find_first_of(fbdChars); + if (fbdMatch != std::string::npos && !(soptype == "*")) { + fprintf(stderr, "illegal character %c detected in optype %s\n", + soptype[fbdMatch], soptype.c_str()); + return 0; + } + if (!(sgeotag == "*" && subcmd != "add")) { + std::string tmp_geotag = eos::common::SanitizeGeoTag(sgeotag); + if (tmp_geotag != sgeotag) { + fprintf(stderr, "%s\n", tmp_geotag.c_str()); + return 0; + } + } + in += ("&mgm.subcmd=disabled" + subcmd); + if (geotag.length()) { + in += ("&mgm.geotag=" + geotag); + } + in += ("&mgm.schedgroup=" + group); + in += ("&mgm.optype=" + optype); + } + if (cmd == "access") { + XrdOucString subcmd = subtokenizer.GetToken(); + XrdOucString geotag, geotag_list, optype; + if ((subcmd != "setdirect") && (subcmd != "showdirect") && + (subcmd != "cleardirect") && (subcmd != "setproxygroup") && + (subcmd != "showproxygroup") && (subcmd != "clearproxygroup")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + const char* token = 0; + if ((token = subtokenizer.GetToken())) { + geotag = token; + } + if ((token = subtokenizer.GetToken())) { + geotag_list = token; + } + in += ("&mgm.subcmd=access" + subcmd); + if (subcmd == "showdirect" || subcmd == "showproxygroup") { + if (geotag.length()) { + if (geotag != "-m" || geotag_list.length()) { + printHelp(); + global_retc = EINVAL; + return 0; + } else { + in += "&mgm.monitoring=1"; + } + } + } else { + if (subcmd == "setdirect" || subcmd == "setproxygroup") { + if (!geotag.length() || !geotag_list.length()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + if (subcmd == "setdirect") { + std::string tmp_list(geotag_list.c_str()); + auto geotags = + eos::common::StringTokenizer::split>( + tmp_list, ','); + tmp_list.clear(); + for (const auto& tag : geotags) { + std::string tmp_tag = eos::common::SanitizeGeoTag(tag); + if (tmp_tag != tag) { + fprintf(stderr, "%s\n", tmp_tag.c_str()); + return 0; + } + } + } + in += ("&mgm.geotaglist=" + geotag_list); + } else { + if (!geotag.length() || geotag_list.length()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + } + std::string tmp_geotag = eos::common::SanitizeGeoTag(geotag.c_str()); + if (tmp_geotag != geotag.c_str()) { + fprintf(stderr, "%s\n", tmp_geotag.c_str()); + return 0; + } + in += ("&mgm.geotag=" + geotag); + } + } + if (subtokenizer.GetToken()) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = output_result(client_command(in, true)); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeGeoschedHelp().c_str()); + } +}; +} // namespace + +void +RegisterGeoschedNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/group-proto-native.cc b/console/commands/native/group-proto-native.cc new file mode 100644 index 0000000000..4ef1ad41c0 --- /dev/null +++ b/console/commands/native/group-proto-native.cc @@ -0,0 +1,168 @@ +// ---------------------------------------------------------------------- +// File: group-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include + +namespace { +std::string MakeGroupHelp() +{ + return "Usage: group ls|rm|set [OPTIONS]\n\n" + " ls [-s] [-g ] [-b|--brief] [-m|-l|--io|--IO] []\n" + " list groups (optional substring match)\n" + " -s : silent, -g : geo depth, -b : brief, -m : monitoring, -l : long\n" + " --io : IO stats per group, --IO : IO stats per filesystem\n\n" + " rm \n" + " remove group\n\n" + " set on|drain|off\n" + " activate/drain/deactivate group\n"; +} + +void ConfigureGroupApp(CLI::App& app) +{ + app.name("group"); + app.description("Group configuration"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeGroupHelp(); + })); +} + +class GroupHelper : public ICmdHelper { +public: + GroupHelper(const GlobalOptions& opts) : ICmdHelper(opts) {} + ~GroupHelper() override = default; + bool + ParseCommand(const char* arg) override + { + eos::console::GroupProto* group = mReq.mutable_group(); + eos::common::StringTokenizer tokenizer(arg); + tokenizer.GetLine(); + std::string token; + if (!tokenizer.NextToken(token)) { + return false; + } + if (token == "ls") { + eos::console::GroupProto_LsProto* ls = group->mutable_ls(); + while (tokenizer.NextToken(token)) { + if (token == "-s") { + mIsSilent = true; + } else if (token == "-g") { + if (!tokenizer.NextToken(token) || + !eos::common::StringTokenizer::IsUnsignedNumber(token)) { + std::cerr << "error: geodepth invalid" << std::endl; + return false; + } + try { + ls->set_outdepth(std::stoi(token)); + } catch (...) { + std::cerr << "error: geodepth must be integer" << std::endl; + return false; + } + } else if (token == "-b" || token == "--brief") { + ls->set_outhost(true); + } else if (token == "-m") { + ls->set_outformat(eos::console::GroupProto_LsProto::MONITORING); + } else if (token == "-l") { + ls->set_outformat(eos::console::GroupProto_LsProto::LISTING); + } else if (token == "--io") { + ls->set_outformat(eos::console::GroupProto_LsProto::IOGROUP); + } else if (token == "--IO") { + ls->set_outformat(eos::console::GroupProto_LsProto::IOFS); + } else if (token.find('-') != 0) { + ls->set_selection(token); + } else { + return false; + } + } + } else if (token == "rm") { + if (!tokenizer.NextToken(token)) { + return false; + } + eos::console::GroupProto_RmProto* rm = group->mutable_rm(); + rm->set_group(token); + } else if (token == "set") { + if (!tokenizer.NextToken(token)) { + return false; + } + eos::console::GroupProto_SetProto* set = group->mutable_set(); + set->set_group(token); + if (!tokenizer.NextToken(token)) { + return false; + } + if (token == "on" || token == "off" || token == "drain") { + set->set_group_state(token); + } else { + return false; + } + } else { + return false; + } + return true; + } +}; + +class GroupProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "group"; + } + const char* + description() const override + { + return "Group configuration"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + GroupHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureGroupApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterGroupProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/health-native.cc b/console/commands/native/health-native.cc new file mode 100644 index 0000000000..54ba037717 --- /dev/null +++ b/console/commands/native/health-native.cc @@ -0,0 +1,54 @@ +// ---------------------------------------------------------------------- +// File: health-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/commands/HealthCommand.hh" +#include +#include + +namespace { +class HealthConsoleCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "health"; + } + const char* + description() const override + { + return "Cluster health check"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + ::HealthCommand cmd(joined.c_str()); + cmd.Execute(); + global_retc = 0; + return 0; + } + void + printHelp() const override + { + } +}; +} // namespace + +void +RegisterHealthNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/info-alias.cc b/console/commands/native/info-alias.cc new file mode 100644 index 0000000000..62b75f43bc --- /dev/null +++ b/console/commands/native/info-alias.cc @@ -0,0 +1,41 @@ +// ---------------------------------------------------------------------- +// File: info-alias.cc +// Purpose: Provide 'info' alias for 'file info ...' +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include + +namespace { +class InfoAliasCommand : public IConsoleCommand { +public: + const char* name() const override { return "info"; } + const char* description() const override { return "Alias for 'file info'"; } + bool requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int run(const std::vector& args, CommandContext& ctx) override + { + IConsoleCommand* fileCmd = CommandRegistry::instance().find("file"); + if (!fileCmd) { + fprintf(stderr, "error: 'file' command not available\n"); + return -1; + } + std::vector forwarded; + forwarded.reserve(args.size() + 1); + forwarded.emplace_back("info"); + forwarded.insert(forwarded.end(), args.begin(), args.end()); + return fileCmd->run(forwarded, ctx); + } + void printHelp() const override { + fprintf(stderr, "Usage: info [options] (alias for 'file info')\n"); + } +}; +} // namespace + +void RegisterInfoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/info-native.cc b/console/commands/native/info-native.cc new file mode 100644 index 0000000000..73a41cc005 --- /dev/null +++ b/console/commands/native/info-native.cc @@ -0,0 +1,80 @@ +// ---------------------------------------------------------------------- +// File: info-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include + +namespace { +class InfoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "info"; + } + const char* + description() const override + { + return "Retrieve file or directory information"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + if (args.empty() || wants_help(args[0].c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + // Build mgm.cmd=fileinfo with filters + XrdOucString path = args[0].c_str(); + if ((!path.beginswith("fid:")) && (!path.beginswith("fxid:")) && + (!path.beginswith("pid:")) && (!path.beginswith("pxid:")) && + (!path.beginswith("inode:"))) { + path = abspath(path.c_str()); + } + XrdOucString in = "mgm.cmd=fileinfo&mgm.path="; + in += path; + // Collect remaining flags/options into mgm.file.info.option + XrdOucString option = ""; + for (size_t i = 1; i < args.size(); ++i) { + XrdOucString tok = args[i].c_str(); + if (tok.length()) { + if (tok == "s") { + option += "silent"; + } else { + option += tok; + } + } + } + if (option.length()) { + in += "&mgm.file.info.option="; + in += option; + } + // If not silent, print output + if (option.find("silent") == STR_NPOS) { + global_retc = + ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + } + return 0; + } + void + printHelp() const override + { + } +}; +} // namespace + +void +RegisterInfoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/inspector-proto-native.cc b/console/commands/native/inspector-proto-native.cc new file mode 100644 index 0000000000..1155a4eb84 --- /dev/null +++ b/console/commands/native/inspector-proto-native.cc @@ -0,0 +1,150 @@ +// ---------------------------------------------------------------------- +// File: inspector-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringTokenizer.hh" +#include "console/CommandFramework.hh" +#include +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include + +namespace { +std::string MakeInspectorHelp() +{ + return "Usage: inspector [-s|--space ] [OPTIONS]\n\n" + "Forward arguments to the inspector space subsystem.\n" + "Options: -c|--current, -l|--last, -m, -p, -e, -C|--cost, -U|--usage,\n" + " -L|--layouts, -B|--birth, -A|--access, -a|--all, -V|--vs, -M|--money\n"; +} + +void ConfigureInspectorApp(CLI::App& app) +{ + app.name("inspector"); + app.description("Run inspector tools"); + app.set_help_flag(""); + app.allow_extras(); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeInspectorHelp(); + })); +} + +class InspectorHelper : public ICmdHelper { +public: + explicit InspectorHelper(const GlobalOptions& opts) : ICmdHelper(opts) + { + mIsAdmin = true; + } + bool + ParseCommand(const char* arg) override + { + eos::common::StringTokenizer tok(arg); + tok.GetLine(); + std::string token; + eos::console::SpaceProto* space = mReq.mutable_space(); + eos::console::SpaceProto_InspectorProto* insp = space->mutable_inspector(); + insp->set_mgmspace("default"); + std::string options; + + while (tok.NextToken(token)) { + if ((token == "-s") || (token == "--space")) { + if (tok.NextToken(token)) { + insp->set_mgmspace(token); + } else { + std::cerr << "error: no space specified" << std::endl; + return false; + } + } else if (token == "-c" || token == "--current") { + options += "c"; + } else if (token == "-l" || token == "--last") { + options += "l"; + } else if (token == "-m") { + options += "m"; + } else if (token == "-p") { + options += "p"; + } else if (token == "-e") { + options += "e"; + } else if (token == "-C" || token == "--cost") { + options += "C"; + } else if (token == "-U" || token == "--usage") { + options += "U"; + } else if (token == "-L" || token == "--layouts") { + options += "L"; + } else if (token == "-B" || token == "--birth") { + options += "B"; + } else if (token == "-A" || token == "--access") { + options += "A"; + } else if (token == "-a" || token == "--all") { + options += "Z"; + } else if (token == "-V" || token == "--vs") { + options += "V"; + } else if (token == "-M" || token == "--money") { + options += "M"; + } else { + return false; + } + } + + insp->set_options(options); + return true; + } +}; + +class InspectorCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "inspector"; + } + const char* + description() const override + { + return "Run inspector tools"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + InspectorHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(true, true); + return 0; + } + void + printHelp() const override + { + CLI::App app; + ConfigureInspectorApp(app); + fprintf(stderr, "%s", app.help().c_str()); + } +}; +} // namespace + +void +RegisterInspectorNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/io-proto-native.cc b/console/commands/native/io-proto-native.cc new file mode 100644 index 0000000000..022b540fc5 --- /dev/null +++ b/console/commands/native/io-proto-native.cc @@ -0,0 +1,418 @@ +// ---------------------------------------------------------------------- +// File: io-proto-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringConversion.hh" +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include "console/commands/helpers/ICmdHelper.hh" +#include +#include +#include +#include + +namespace { + +std::string MakeIoHelp() +{ + std::ostringstream oss; + oss << "Usage: io stat|enable|disable|report|ns|shaping [OPTIONS]\n\n" + << "io stat [-l] [-a] [-m] [-n] [-t] [-d] [-x] [--ss] [--sa] [--si] : print io " + "statistics\n" + << "\t -l : show summary information (this is the default if -a,-t,-d,-x is not " + "selected)\n" + << "\t -a : break down by uid/gid\n" + << "\t -m : print in = monitoring format\n" + << "\t -n : print numerical uid/gids\n" + << "\t -t : print top user stats\n" + << "\t -d : break down by domains\n" + << "\t -x : break down by application\n" + << "\t --ss : show table with transfer sample statistics\n" + << "\t --sa : start collection of statistics given number of seconds ago\n" + << "\t --si : collect statistics over given interval of seconds\n" + << "\t Note: this tool shows data for finished transfers only (using storage node " + "reports)\n" + << "\t Example: asking for data of finished transfers which were transferred " + "during interval [now - 180s, now - 120s]:\n" + << "\t eos io stat -x --sa 120 --si 60\n\n" + << "io enable [-r] [-p] [-n] [--udp
] : enable collection of io " + "statistics\n" + << "\t no arg. : start the collection thread\n" + << "\t -r : enable collection of io reports\n" + << "\t -p : enable popularity accounting\n" + << "\t -n : enable report namespace\n" + << "\t --udp
: add a UDP message target for io UDP packets\n\n" + << "io disable [-r] [-p] [-n] [--udp
] : disable collection of io " + "statistics\n" + << "\t no arg. : stop the collection thread\n" + << "\t -r : disable collection of io reports\n" + << "\t -p : disable popularity accounting\n" + << "\t -n : disable report namespace\n" + << "\t --udp
: remove a UDP message target for io UDP packets\n\n" + << "io report : show contents of report namespace for \n\n" + << "io ns [-a] [-n] [-b] [-100|-1000|-10000] [-w] [-f] : show namespace IO ranking " + "(popularity)\n" + << "\t -a : don't limit the output list\n" + << "\t -n : show ranking by number of accesses\n" + << "\t -b : show ranking by number of bytes\n" + << "\t -100 : show the first 100 in the ranking\n" + << "\t -1000 : show the first 1000 in the ranking\n" + << "\t -10000 : show the first 10000 in the ranking\n" + << "\t -w : show history for the last 7 days\n" + << "\t -f : show the 'hotfiles' which are the files with highest number of " + "present file opens\n\n" + << "io shaping ls|enable|disable|policy|config [options...] : interact with the " + "Traffic Shaping engine\n\n" + << " SUBCOMMANDS\n" + << " ls [options...] : view real-time IO rates and shaping status\n" + << "\t --apps : show rates by application\n" + << "\t --users : show rates by user (uid)\n" + << "\t --groups : show rates by group (gid)\n" + << "\t --nodes : show rates by storage node (FST)\n" + << "\t --json : output in JSON format\n" + << "\t --sys : include meta statistics about Traffic Shaping system\n" + << "\t --window <1|5|15|60|300> : time window in seconds for SMA (default 60)\n\n" + << " enable : globally enable traffic shaping\n" + << " disable : globally disable traffic shaping\n\n" + << " policy [action] [options...] : manage shaping limits and reservations\n" + << "\t action 'ls' : list configured policies\n" + << "\t --apps, --users, --groups, --controller, --json\n" + << "\t action 'set' : configure policy (--app|--uid|--gid, --limit-read, " + "--limit-write, etc.)\n" + << "\t action 'rm' : remove a configured policy\n\n" + << " config ls|set : manage traffic shaping thread configurations\n"; + return oss.str(); +} + +//------------------------------------------------------------------------------ +// Build full CLI11 app with all io subcommands and populate proto +//------------------------------------------------------------------------------ +bool +BuildAndParseIoApp(const std::string& input, eos::console::IoProto* io) +{ + CLI::App app{"io"}; + app.require_subcommand(1); + + // stat + auto* stat_cmd = app.add_subcommand("stat", "Print IO statistics"); + stat_cmd->add_flag("-l", "Show summary"); + stat_cmd->add_flag("-a", "Break down by uid/gid"); + stat_cmd->add_flag("-m", "Monitoring format"); + stat_cmd->add_flag("-n", "Numerical uid/gids"); + stat_cmd->add_flag("-t", "Top user stats"); + stat_cmd->add_flag("-d", "Break down by domains"); + stat_cmd->add_flag("-x", "Break down by application"); + stat_cmd->add_flag("--ss", "Transfer sample statistics"); + stat_cmd->add_option("--sa", "Seconds ago")->type_name("SEC"); + stat_cmd->add_option("--si", "Time interval in seconds")->type_name("SEC"); + stat_cmd->callback([stat_cmd, io]() { + auto* stat = io->mutable_stat(); + stat->set_summary(stat_cmd->count("-l") > 0); + stat->set_details(stat_cmd->count("-a") > 0); + stat->set_monitoring(stat_cmd->count("-m") > 0); + stat->set_numerical(stat_cmd->count("-n") > 0); + stat->set_top(stat_cmd->count("-t") > 0); + stat->set_domain(stat_cmd->count("-d") > 0); + stat->set_apps(stat_cmd->count("-x") > 0); + stat->set_sample_stat(stat_cmd->count("--ss") > 0); + if (stat_cmd->count("--sa")) { + stat->set_time_ago(stat_cmd->get_option("--sa")->as()); + } + if (stat_cmd->count("--si")) { + stat->set_time_interval(stat_cmd->get_option("--si")->as()); + } + }); + + // ns + auto* ns_cmd = app.add_subcommand("ns", "Show namespace IO ranking"); + ns_cmd->add_flag("-m", "Monitoring format"); + ns_cmd->add_flag("-b", "Rank by bytes"); + ns_cmd->add_flag("-n", "Rank by access count"); + ns_cmd->add_flag("-w", "Last 7 days"); + ns_cmd->add_flag("-f", "Hotfiles"); + ns_cmd->allow_non_standard_option_names(); + ns_cmd->add_flag("-a", "Don't limit output"); + ns_cmd->add_flag("-100", "Top 100"); + ns_cmd->add_flag("-1000", "Top 1000"); + ns_cmd->add_flag("-10000", "Top 10000"); + ns_cmd->callback([ns_cmd, io]() { + auto* ns = io->mutable_ns(); + ns->set_monitoring(ns_cmd->count("-m") > 0); + ns->set_rank_by_byte(ns_cmd->count("-b") > 0); + ns->set_rank_by_access(ns_cmd->count("-n") > 0); + ns->set_last_week(ns_cmd->count("-w") > 0); + ns->set_hotfiles(ns_cmd->count("-f") > 0); + if (ns_cmd->count("-a") > 0) + ns->set_count(eos::console::IoProto_NsProto::ALL); + else if (ns_cmd->count("-100") > 0) + ns->set_count(eos::console::IoProto_NsProto::ONEHUNDRED); + else if (ns_cmd->count("-1000") > 0) + ns->set_count(eos::console::IoProto_NsProto::ONETHOUSAND); + else if (ns_cmd->count("-10000") > 0) + ns->set_count(eos::console::IoProto_NsProto::TENTHOUSAND); + }); + + // report + auto* report_cmd = app.add_subcommand("report", "Show report namespace for path"); + report_cmd->add_option("path", "Path to report")->required(); + report_cmd->callback([report_cmd, io]() { + io->mutable_report()->set_path(report_cmd->get_option("path")->as()); + }); + + // enable + auto* enable_cmd = app.add_subcommand("enable", "Enable IO statistics collection"); + enable_cmd->add_flag("-r", "Enable reports"); + enable_cmd->add_flag("-p", "Enable popularity"); + enable_cmd->add_flag("-n", "Enable report namespace"); + enable_cmd->add_option("--udp", "UDP target address")->type_name("ADDR"); + enable_cmd->callback([enable_cmd, io]() { + auto* en = io->mutable_enable(); + en->set_switchx(true); + en->set_reports(enable_cmd->count("-r") > 0); + en->set_popularity(enable_cmd->count("-p") > 0); + en->set_namespacex(enable_cmd->count("-n") > 0); + if (enable_cmd->count("--udp")) + en->set_upd_address(enable_cmd->get_option("--udp")->as()); + }); + + // disable + auto* disable_cmd = app.add_subcommand("disable", "Disable IO statistics collection"); + disable_cmd->add_flag("-r", "Disable reports"); + disable_cmd->add_flag("-p", "Disable popularity"); + disable_cmd->add_flag("-n", "Disable report namespace"); + disable_cmd->add_option("--udp", "UDP target address to remove")->type_name("ADDR"); + disable_cmd->callback([disable_cmd, io]() { + auto* en = io->mutable_enable(); + en->set_switchx(false); + en->set_reports(disable_cmd->count("-r") > 0); + en->set_popularity(disable_cmd->count("-p") > 0); + en->set_namespacex(disable_cmd->count("-n") > 0); + if (disable_cmd->count("--udp")) + en->set_upd_address(disable_cmd->get_option("--udp")->as()); + }); + + // shaping (nested subcommands) + auto* shaping_cmd = app.add_subcommand("shaping", "Traffic Shaping engine"); + shaping_cmd->require_subcommand(1); + eos::console::IoProto_ShapingProto* shaping_proto = io->mutable_shaping(); + + auto* shaping_ls = shaping_cmd->add_subcommand("ls", "View real-time IO rates"); + auto* grp = shaping_ls->add_option_group("Grouping")->require_option(0, 1); + grp->add_flag("--apps", "Show rates by application"); + grp->add_flag("--users", "Show rates by user"); + grp->add_flag("--groups", "Show rates by group"); + grp->add_flag("--nodes", "Show rates by storage node"); + shaping_ls->add_flag("--json", "JSON output"); + shaping_ls->add_flag("--sys", "Include system stats"); + shaping_ls->add_option("--window", "Time window (1|5|15|60|300)") + ->check(CLI::IsMember({"1", "5", "15", "60", "300"})) + ->default_val("60"); + shaping_ls->callback([shaping_ls, shaping_proto]() { + auto* action = shaping_proto->mutable_list(); + bool su = shaping_ls->count("--users") > 0; + bool sg = shaping_ls->count("--groups") > 0; + bool sn = shaping_ls->count("--nodes") > 0; + action->set_show_apps(shaping_ls->count("--apps") > 0 || (!su && !sg && !sn)); + action->set_show_users(su); + action->set_show_groups(sg); + action->set_show_nodes(sn); + action->set_json_output(shaping_ls->count("--json") > 0); + action->set_system_stats(shaping_ls->count("--sys") > 0); + action->set_time_window_seconds( + static_cast(std::stoul(shaping_ls->get_option("--window")->as()))); + }); + + shaping_cmd->add_subcommand("enable", "Enable traffic shaping") + ->callback([shaping_proto]() { shaping_proto->mutable_enable(); }); + shaping_cmd->add_subcommand("disable", "Disable traffic shaping") + ->callback([shaping_proto]() { shaping_proto->mutable_disable(); }); + + auto* policy_cmd = shaping_cmd->add_subcommand("policy", "Manage shaping policies"); + policy_cmd->require_subcommand(1); + + auto* policy_ls = policy_cmd->add_subcommand("ls", "List policies"); + policy_ls->add_flag("--apps"); + policy_ls->add_flag("--users"); + policy_ls->add_flag("--groups"); + policy_ls->add_flag("--controller"); + policy_ls->add_flag("--json"); + policy_ls->callback([policy_ls, shaping_proto]() { + auto* a = shaping_proto->mutable_policy()->mutable_list(); + a->set_filter_apps(policy_ls->count("--apps") > 0); + a->set_filter_users(policy_ls->count("--users") > 0); + a->set_filter_groups(policy_ls->count("--groups") > 0); + a->set_show_controller_limits(policy_ls->count("--controller") > 0); + a->set_json_output(policy_ls->count("--json") > 0); + }); + + auto* policy_set = policy_cmd->add_subcommand("set", "Set policy"); + auto* tgrp = policy_set->add_option_group("Target")->require_option(1); + tgrp->add_option("--app", "Application"); + tgrp->add_option("--uid", "User ID"); + tgrp->add_option("--gid", "Group ID"); + auto* pgrp = policy_set->add_option_group("Params")->require_option(1, 6); + pgrp->add_option("--limit-read"); + pgrp->add_option("--limit-write"); + pgrp->add_option("--reservation-read"); + pgrp->add_option("--reservation-write"); + pgrp->add_option("--controller-limit-read"); + pgrp->add_option("--controller-limit-write"); + auto* pe = pgrp->add_flag("--enable"); + auto* pd = pgrp->add_flag("--disable"); + pe->excludes(pd); + policy_set->callback([policy_set, shaping_proto]() { + auto* a = shaping_proto->mutable_policy()->mutable_set(); + if (policy_set->count("--app")) + a->set_app(policy_set->get_option("--app")->as()); + if (policy_set->count("--uid")) + a->set_uid(policy_set->get_option("--uid")->as()); + if (policy_set->count("--gid")) + a->set_gid(policy_set->get_option("--gid")->as()); + auto parse_rate = [](const std::string& s) -> uint64_t { + uint64_t n = 0; + eos::common::StringConversion::GetSizeFromString(s, n); + return n; + }; + if (policy_set->count("--limit-read")) + a->set_limit_read_bytes_per_sec( + parse_rate(policy_set->get_option("--limit-read")->as())); + if (policy_set->count("--limit-write")) + a->set_limit_write_bytes_per_sec( + parse_rate(policy_set->get_option("--limit-write")->as())); + if (policy_set->count("--reservation-read")) + a->set_reservation_read_bytes_per_sec( + parse_rate(policy_set->get_option("--reservation-read")->as())); + if (policy_set->count("--reservation-write")) + a->set_reservation_write_bytes_per_sec( + parse_rate(policy_set->get_option("--reservation-write")->as())); + if (policy_set->count("--controller-limit-read")) + a->set_controller_limit_read_bytes_per_sec( + parse_rate(policy_set->get_option("--controller-limit-read")->as())); + if (policy_set->count("--controller-limit-write")) + a->set_controller_limit_write_bytes_per_sec( + parse_rate(policy_set->get_option("--controller-limit-write")->as())); + if (policy_set->count("--enable")) + a->set_is_enabled(true); + else if (policy_set->count("--disable")) + a->set_is_enabled(false); + }); + + auto* policy_rm = policy_cmd->add_subcommand("rm", "Remove policy"); + auto* rgrp = policy_rm->add_option_group("Target")->require_option(1); + rgrp->add_option("--app"); + rgrp->add_option("--uid"); + rgrp->add_option("--gid"); + policy_rm->callback([policy_rm, shaping_proto]() { + auto* a = shaping_proto->mutable_policy()->mutable_remove(); + if (policy_rm->count("--app")) + a->set_app(policy_rm->get_option("--app")->as()); + if (policy_rm->count("--uid")) + a->set_uid(policy_rm->get_option("--uid")->as()); + if (policy_rm->count("--gid")) + a->set_gid(policy_rm->get_option("--gid")->as()); + }); + + auto* config_cmd = shaping_cmd->add_subcommand("config", "Shaping config"); + config_cmd->require_subcommand(1); + config_cmd->add_subcommand("ls", "List config") + ->callback([shaping_proto]() { shaping_proto->mutable_config()->mutable_list(); }); + auto* config_set = config_cmd->add_subcommand("set", "Set config"); + config_set->require_option(1, 4); + config_set->add_option("--estimators-period"); + config_set->add_option("--policy-period"); + config_set->add_option("--report-period"); + config_set->add_option("--system-window"); + config_set->callback([config_set, shaping_proto]() { + auto* a = shaping_proto->mutable_config()->mutable_set(); + if (config_set->count("--estimators-period")) + a->set_update_estimators_thread_period_ms( + config_set->get_option("--estimators-period")->as()); + if (config_set->count("--policy-period")) + a->set_fst_io_policy_update_thread_period_ms( + config_set->get_option("--policy-period")->as()); + if (config_set->count("--report-period")) + a->set_fst_io_stats_reporting_thread_period_ms( + config_set->get_option("--report-period")->as()); + if (config_set->count("--system-window")) + a->set_system_stats_time_window_seconds( + config_set->get_option("--system-window")->as()); + }); + + std::string to_parse = "io " + input; + try { + app.parse(to_parse, true); + } catch (const CLI::ParseError& e) { + app.exit(e); + return false; + } + return true; +} + +class IoHelper : public ICmdHelper { +public: + IoHelper(const GlobalOptions& opts) : ICmdHelper(opts) {} + ~IoHelper() override = default; + bool + ParseCommand(const char* arg) override + { + return BuildAndParseIoApp(arg, mReq.mutable_io()); + } +}; + +class IoProtoCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "io"; + } + const char* + description() const override + { + return "IO Interface"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext&) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + if (args[i].find(' ') != std::string::npos) + oss << std::quoted(args[i]); + else + oss << args[i]; + } + std::string joined = oss.str(); + if (wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + IoHelper helper(gGlobalOpts); + if (!helper.ParseCommand(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + global_retc = helper.Execute(); + return 0; + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeIoHelp().c_str()); + } +}; +} // namespace + +void +RegisterIoProtoNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/license-native.cc b/console/commands/native/license-native.cc new file mode 100644 index 0000000000..da9d052f36 --- /dev/null +++ b/console/commands/native/license-native.cc @@ -0,0 +1,47 @@ +// ---------------------------------------------------------------------- +// File: license-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include +#include + +extern const char* license; + +namespace { +class LicenseCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "license"; + } + const char* + description() const override + { + return "Show EOS license information"; + } + bool + requiresMgm(const std::string&) const override + { + return false; + } + int + run(const std::vector&, CommandContext&) override + { + fprintf(stdout, "%s", license); + global_retc = 0; + return 0; + } + void + printHelp() const override + { + } +}; +} // namespace + +void +RegisterLicenseNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/ln-cmd-native.cc b/console/commands/native/ln-cmd-native.cc new file mode 100644 index 0000000000..2cb3d57fa5 --- /dev/null +++ b/console/commands/native/ln-cmd-native.cc @@ -0,0 +1,109 @@ +// ---------------------------------------------------------------------- +// File: ln-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include + +namespace { +std::string MakeLnHelp() +{ + return "Usage: ln [-f] \n" + "Create a symbolic link from to .\n" + " -f force overwrite if already exists\n"; +} + +void ConfigureLnApp(CLI::App& app, std::string& link, std::string& target, + bool& force) +{ + app.name("ln"); + app.description("Create a symbolic link"); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App*, std::string, CLI::AppFormatMode) { + return MakeLnHelp(); + })); + app.add_flag("-f,--force", force, "force overwrite if link already exists"); + app.add_option("link", link, "link path")->required(); + app.add_option("target", target, "target path")->required(); +} + +class LnCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "ln"; + } + const char* + description() const override + { + return "Create a symbolic link"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + int + run(const std::vector& args, CommandContext& ctx) override + { + std::ostringstream oss; + for (size_t i = 0; i < args.size(); ++i) { + if (i) + oss << ' '; + oss << args[i]; + } + std::string joined = oss.str(); + if (args.size() < 2 || wants_help(joined.c_str())) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + IConsoleCommand* fileCmd = CommandRegistry::instance().find("file"); + if (!fileCmd) { + fprintf(stderr, "error: 'file' command not available\n"); + global_retc = EINVAL; + return 0; + } + + CLI::App app; + std::string link; + std::string target; + bool force = false; + ConfigureLnApp(app, link, target, force); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + std::vector fargs = {"symlink", link, target}; + if (force) + fargs.insert(fargs.begin() + 1, "-f"); + return fileCmd->run(fargs, ctx); + } + void + printHelp() const override + { + fprintf(stderr, "%s", MakeLnHelp().c_str()); + } +}; +} // namespace + +void +RegisterLnNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/ls-cmd-native.cc b/console/commands/native/ls-cmd-native.cc new file mode 100644 index 0000000000..650866ad8f --- /dev/null +++ b/console/commands/native/ls-cmd-native.cc @@ -0,0 +1,394 @@ +// ---------------------------------------------------------------------- +// File: ls-native.cc +// ---------------------------------------------------------------------- + +#include "common/StringConversion.hh" +#include "console/CommandFramework.hh" +#include +#include "namespace/utils/Mode.hh" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::string MakeLsHelp(const CLI::App* app) +{ + std::ostringstream oss; + const std::string& name = app->get_name(); + oss << "Usage: " << (name.empty() ? "ls" : name) + << " [OPTION]... [--no-globbing] [PATH]...\n"; + const std::string desc = app->get_description(); + if (!desc.empty()) { + oss << desc << "\n"; + } + oss << "\nOptions:\n"; + + std::vector> lines; + size_t max_name = 0; + for (const auto* opt : app->get_options()) { + if (!opt || !opt->nonpositional()) { + continue; + } + std::string opt_name = opt->get_name(false, true); + if (opt_name.empty()) { + continue; + } + for (size_t i = 0; i + 1 < opt_name.size(); ++i) { + if (opt_name[i] == ',' && opt_name[i + 1] != ' ') { + opt_name.insert(i + 1, " "); + ++i; + } + } + std::string opt_desc = opt->get_description(); + max_name = std::max(max_name, opt_name.size()); + lines.emplace_back(std::move(opt_name), std::move(opt_desc)); + } + + for (const auto& line : lines) { + oss << " " << line.first; + if (!line.second.empty()) { + size_t pad = (max_name > line.first.size()) ? (max_name - line.first.size()) : 0; + oss << std::string(pad + 2, ' ') << line.second; + } + oss << "\n"; + } + + oss << "\nNotes:\n" + << " -lh: show long listing with readable sizes\n" + << " path=file:... : list on a local file system\n" + << " path=root:... : list on a plain XRootD server (does not work on native XRootD clusters)\n" + << " path=... : all other paths are considered to be EOS paths!\n"; + return oss.str(); +} + +void ConfigureLsApp(CLI::App& app, + bool& opt_l, + bool& opt_y, + bool& opt_a, + bool& opt_i, + bool& opt_c, + bool& opt_n, + bool& opt_F, + bool& opt_s, + bool& opt_no_globbing) +{ + app.name("ls"); + app.description("list directory "); + app.set_help_flag(""); + app.formatter(std::make_shared( + [](const CLI::App* app, std::string, CLI::AppFormatMode) { + return MakeLsHelp(app); + })); + + app.add_flag("-l", opt_l, "show long listing"); + app.add_flag("-y", opt_y, "show long listing with backend(tape) status"); + // Note: '-lh' was accepted historically; '-h' alone shows help in legacy + app.add_flag("-a", opt_a, "show hidden files"); + app.add_flag("-i", opt_i, "add inode information"); + app.add_flag("-c", opt_c, "add checksum value (implies -l)"); + app.add_flag("-n", opt_n, "show numerical user/group ids"); + app.add_flag("-F", opt_F, "append indicator '/' to directories"); + app.add_flag("-s", opt_s, "checks only if the directory exists without listing"); + app.add_flag("-N,--no-globbing", opt_no_globbing, "disables globbing"); +} + +class LsCommand : public IConsoleCommand { +public: + const char* + name() const override + { + return "ls"; + } + const char* + description() const override + { + return "List a directory"; + } + bool + requiresMgm(const std::string& args) const override + { + return !wants_help(args.c_str()); + } + + int + run(const std::vector& args, CommandContext& ctx) override + { + // Setup CLI11 parser (ignore unknown options) + CLI::App app; + app.allow_extras(); + + bool opt_l = false; + bool opt_y = false; + bool opt_a = false; + bool opt_i = false; + bool opt_c = false; + bool opt_n = false; + bool opt_F = false; + bool opt_s = false; + bool opt_no_globbing = false; + + ConfigureLsApp(app, opt_l, opt_y, opt_a, opt_i, opt_c, opt_n, opt_F, opt_s, + opt_no_globbing); + + std::vector positionals; + app.add_option("path", positionals); + + std::vector cli_args = args; + std::reverse(cli_args.begin(), cli_args.end()); + try { + app.parse(cli_args); + } catch (const CLI::ParseError&) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + // Help handling consistent with legacy (-h or --help) + if (!args.empty() && (args[0] == "-h" || args[0] == "--help")) { + printHelp(); + global_retc = EINVAL; + return 0; + } + + // Build mgm.option string + std::string option; + auto add_flag = [&](char c) { + option += '-'; + option.push_back(c); + }; + if (opt_l) + add_flag('l'); + if (opt_y) + add_flag('y'); + if (opt_a) + add_flag('a'); + if (opt_i) + add_flag('i'); + if (opt_c) { + add_flag('c'); + if (!opt_l) + add_flag('l'); + } + if (opt_n) + add_flag('n'); + if (opt_F) + add_flag('F'); + if (opt_s) + add_flag('s'); + if (opt_no_globbing) { + option += "-N"; + } + + // Determine path from positionals (join to allow spaces) + std::ostringstream pathoss; + for (size_t i = 0; i < positionals.size(); ++i) { + if (i) + pathoss << ' '; + pathoss << positionals[i]; + } + std::string path = pathoss.str(); + if (path.empty()) + path = gPwd.c_str(); + + // Unescape blanks (legacy behaviour: replace "\\ " -> " ") + { + std::string replaced; + replaced.reserve(path.size()); + for (size_t i = 0; i < path.size(); ++i) { + if (path[i] == '\\' && i + 1 < path.size() && path[i + 1] == ' ') { + replaced.push_back(' '); + ++i; + } else + replaced.push_back(path[i]); + } + path.swap(replaced); + } + + XrdOucString xpath = path.c_str(); + + // S3 scheme handling (legacy behaviour): as3: + if (xpath.beginswith("as3:")) { + XrdOucString hostport; + XrdOucString protocol; + XrdOucString sPath; + const char* v = 0; + if (!(v = eos::common::StringConversion::ParseUrl(xpath.c_str(), protocol, + hostport))) { + fprintf(stderr, "error: illegal url <%s>\n", xpath.c_str()); + global_retc = EINVAL; + return 0; + } + sPath = v; + if (hostport.length()) + setenv("S3_HOSTNAME", hostport.c_str(), 1); + + XrdOucString envString = xpath; + int qpos = 0; + if ((qpos = envString.find("?")) != STR_NPOS) { + envString.erase(0, qpos + 1); + XrdOucEnv env(envString.c_str()); + if (env.Get("s3.key")) + setenv("S3_SECRET_ACCESS_KEY", env.Get("s3.key"), 1); + if (env.Get("s3.id")) + setenv("S3_ACCESS_KEY_ID", env.Get("s3.id"), 1); + xpath.erase(xpath.find("?")); + sPath.erase(sPath.find("?")); + } + const char* cstr = getenv("S3_ACCESS_KEY"); + if (cstr) + setenv("S3_SECRET_ACCESS_KEY", cstr, 1); + cstr = getenv("S3_ACESSS_ID"); + if (cstr) + setenv("S3_ACCESS_KEY_ID", cstr, 1); + if (!getenv("S3_ACCESS_KEY_ID") || !getenv("S3_HOSTNAME") || + !getenv("S3_SECRET_ACCESS_KEY")) { + fprintf(stderr, "error: you have to set the S3 environment variables " + "S3_ACCESS_KEY_ID | S3_ACCESS_ID, S3_HOSTNAME (or use " + "a URI), S3_SECRET_ACCESS_KEY | S3_ACCESS_KEY\n"); + exit(-1); + } + XrdOucString s3env; + s3env = "env S3_ACCESS_KEY_ID="; + s3env += getenv("S3_ACCESS_KEY_ID"); + s3env += " S3_HOSTNAME="; + s3env += getenv("S3_HOSTNAME"); + s3env += " S3_SECRET_ACCESS_KEY="; + s3env += getenv("S3_SECRET_ACCESS_KEY"); + XrdOucString s3arg = sPath.c_str(); + XrdOucString listcmd = "bash -c \""; + listcmd += s3env; + listcmd += " s3 list "; + listcmd += s3arg; + listcmd += " "; + listcmd += "\""; + global_retc = system(listcmd.c_str()); + return 0; + } + + // Local file or plain XRootD path handling + if (xpath.beginswith("file:") || xpath.beginswith("root:")) { + bool isXrd = xpath.beginswith("root:"); + XrdOucString protocol; + XrdOucString hostport; + XrdOucString sPath; + const char* v = 0; + if (!(v = eos::common::StringConversion::ParseUrl(xpath.c_str(), protocol, + hostport))) { + global_retc = EINVAL; + return 0; + } + sPath = v; + std::string Path = v; + if (sPath == "" && (protocol == "file")) { + sPath = getenv("PWD"); + Path = getenv("PWD"); + if (!sPath.endswith("/")) { + sPath += "/"; + Path += "/"; + } + } + XrdOucString url = ""; + eos::common::StringConversion::CreateUrl( + protocol.c_str(), hostport.c_str(), Path.c_str(), url); + DIR* dir = + (isXrd) ? XrdPosixXrootd::Opendir(url.c_str()) : opendir(url.c_str()); + if (dir) { + struct dirent* entry; + while ( + (entry = (isXrd) ? XrdPosixXrootd::Readdir(dir) : readdir(dir))) { + struct stat buf; + XrdOucString curl = ""; + XrdOucString cpath = Path.c_str(); + cpath += entry->d_name; + eos::common::StringConversion::CreateUrl( + protocol.c_str(), hostport.c_str(), cpath.c_str(), curl); + if (option.find("a") == std::string::npos) { + if (entry->d_name[0] == '.') + continue; + } + if (!((isXrd) ? XrdPosixXrootd::Stat(curl.c_str(), &buf) + : stat(curl.c_str(), &buf))) { + if (option.find("l") == std::string::npos) { + fprintf(stdout, "%s\n", entry->d_name); + } else { + char t_creat[14]; + char modestr[11]; + eos::modeToBuffer(buf.st_mode, modestr); + XrdOucString suid = ""; + suid += (int)buf.st_uid; + XrdOucString sgid = ""; + sgid += (int)buf.st_gid; + XrdOucString sizestring = ""; + struct tm* t_tm; + struct tm t_tm_local; + t_tm = localtime_r(&buf.st_ctime, &t_tm_local); + strftime(t_creat, 13, "%b %d %H:%M", t_tm); + XrdOucString dirmarker = ""; + if (option.find("F") != std::string::npos) + dirmarker = "/"; + if (modestr[0] != 'd') + dirmarker = ""; + fprintf(stdout, "%s %3d %-8.8s %-8.8s %12s %s %s%s\n", modestr, + (int)buf.st_nlink, suid.c_str(), sgid.c_str(), + eos::common::StringConversion::GetSizeString( + sizestring, (unsigned long long)buf.st_size), + t_creat, entry->d_name, dirmarker.c_str()); + } + } + } + (isXrd) ? XrdPosixXrootd::Closedir(dir) : closedir(dir); + } + global_retc = 0; + return 0; + } + + // EOS path via MGM + XrdOucString ap = abspath(xpath.c_str()); + if (strlen(ap.c_str()) >= FILENAME_MAX) { + fprintf(stderr, "error: path length longer than %i bytes", FILENAME_MAX); + global_retc = EINVAL; + return 0; + } + XrdOucString esc = + eos::common::StringConversion::curl_escaped(ap.c_str()).c_str(); + XrdOucString in = "mgm.cmd=ls"; + in += "&mgm.path="; + in += esc; + in += "&eos.encodepath=1"; + in += "&mgm.option="; + in += option.c_str(); + global_retc = ctx.outputResult(ctx.clientCommand(in, false, nullptr), true); + return 0; + } + + void + printHelp() const override + { + CLI::App app; + bool opt_l = false; + bool opt_y = false; + bool opt_a = false; + bool opt_i = false; + bool opt_c = false; + bool opt_n = false; + bool opt_F = false; + bool opt_s = false; + bool opt_no_globbing = false; + ConfigureLsApp(app, opt_l, opt_y, opt_a, opt_i, opt_c, opt_n, opt_F, opt_s, + opt_no_globbing); + const std::string help = app.help(); + fprintf(stderr, "%s", help.c_str()); + } +}; +} // namespace + +void +RegisterLsNativeCommand() +{ + CommandRegistry::instance().reg(std::make_unique()); +} diff --git a/console/commands/native/ls-compat.cc b/console/commands/native/ls-compat.cc new file mode 100644 index 0000000000..21433e424f --- /dev/null +++ b/console/commands/native/ls-compat.cc @@ -0,0 +1,48 @@ +// ---------------------------------------------------------------------- +// File: ls-compat.cc +// Purpose: Provide legacy com_ls symbol delegating to native LsCommand +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include + +int +com_ls(char* arg) +{ + std::vector argv; + if (arg && *arg) { + std::string s(arg); + std::string cur; + for (size_t i = 0; i < s.size(); ++i) { + if (s[i] == ' ') { + if (!cur.empty()) { + argv.push_back(cur); + cur.clear(); + } + } else { + cur.push_back(s[i]); + } + } + if (!cur.empty()) + argv.push_back(cur); + } + + CommandContext ctx; + ctx.serverUri = serveruri.c_str(); + ctx.globalOpts = &gGlobalOpts; + ctx.json = json; + ctx.silent = silent; + ctx.interactive = interactive; + ctx.timing = timing; + ctx.userRole = user_role.c_str(); + ctx.groupRole = group_role.c_str(); + ctx.clientCommand = &client_command; + ctx.outputResult = &output_result; + + IConsoleCommand* cmd = CommandRegistry::instance().find("ls"); + if (!cmd) + return -1; + return cmd->run(argv, ctx); +} diff --git a/console/commands/native/map-cmd-native.cc b/console/commands/native/map-cmd-native.cc new file mode 100644 index 0000000000..7bf3f17d7c --- /dev/null +++ b/console/commands/native/map-cmd-native.cc @@ -0,0 +1,159 @@ +// ---------------------------------------------------------------------- +// File: map-native.cc +// ---------------------------------------------------------------------- + +#include "console/CommandFramework.hh" +#include "console/ConsoleMain.hh" +#include +#include +#include +#include +#include +#include + +namespace { +std::string MakeMapHelp() +{ + return "Usage: map [OPTIONS] ls|link|unlink ...\n\n" + "'[eos] map ..' provides a namespace mapping interface for " + "directories in EOS.\n\n" + "Subcommands:\n" + " ls list all defined mappings\n" + " link create a symbolic link from " + "source to destination\n" + " unlink remove symbolic link from " + "source\n\n" + "Options:\n" + " -