feat: configurator tool script

configuration-tool.sh

@@ -0,0 +1,343 @@
+#!/bin/bash
+# filepath: configuration-tool.sh
+
+# SC4S Configuration Tool
+# Generates a customized env_file based on user requirements
+
+set -e
+
+# Colors for better UX
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Initialize variables
+OUTPUT_FILE="env_file"
+SPLUNK_URL=""
+HEC_TOKEN=""
+TLS_VERIFY="yes"
+EXPECTED_EPS=1000
+PROTOCOL="both"
+
+SC4S_SOURCE_LISTEN_UDP_SOCKETS=2
+SC4S_SOURCE_UDP_FETCH_LIMIT=1000
+SC4S_ENABLE_EBPF="no"
+SC4S_EBPF_NO_SOCKETS=4
+
+PARALLELIZE="no"
+SC4S_PARALLELIZE_NO_PARTITION=4
+SC4S_SOURCE_TCP_IW_USE="no"
+SC4S_SOURCE_TCP_IW_SIZE=1000000
+
+SC4S_SOURCE_UDP_SO_RCVBUFF=-1
+SC4S_SOURCE_TCP_SO_RCVBUFF=-1
+
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE="yes"
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE="no"
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE=10241024
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH=15000
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE=53687091200
+
+echo ""
+echo "${BLUE}================================================${NC}"
+echo "${BLUE}    SC4S Configuration Tool${NC}"
+echo "${BLUE}================================================${NC}"
+echo ""
+echo "This tool will help you generate an optimized SC4S configuration"
+echo "based on your requirements."
+echo ""
+
+# Function to ask yes/no questions
+ask_yes_no() {
+    local question="$1"
+    local default="$2"
+    local response
+
+    while true; do
+        if [[ "$default" == "yes" ]]; then
+            read -p "$question [Y/n]: " response
+            response=${response:-y}
+        else
+            read -p "$question [y/N]: " response
+            response=${response:-n}
+        fi
+
+        case "$response" in
+            [Yy]*|yes*|Yes*|YES* ) echo "yes"; break;;
+            [Nn]*|no*|No*|NO* ) echo "no"; break;;
+            * ) echo "Please answer yes or no.";;
+        esac
+    done
+}
+
+echo "${GREEN}=== Splunk Configuration ===${NC}"
+
+# Splunk HEC URL
+read -p "Enter your Splunk HEC URL (e.g., https://your.splunk.instance:8088): " SPLUNK_URL
+
+# HEC Token
+read -p "Enter your Splunk HEC Token: " HEC_TOKEN
+
+# TLS Verification
+TLS_VERIFY=$(ask_yes_no "Verify SSL/TLS certificates?" "yes")
+
+echo ""
+echo "${GREEN}=== Performance Configuration ===${NC}"
+
+# Protocol selection
+echo ""
+echo "Protocol optimisation:"
+echo "1) UDP only (faster, may lose messages)"
+echo "2) TCP only (reliable, slower)"
+echo "3) Both UDP and TCP (default)"
+read -p "Select protocol [3]: " protocol_choice
+protocol_choice=${protocol_choice:-3}
+case "$protocol_choice" in
+    1) PROTOCOL="udp";;
+    2) PROTOCOL="tcp";;
+    3) PROTOCOL="both";;
+    *) PROTOCOL="both";;
+esac
+
+# # Expected EPS
+# read -p "Expected events per second (EPS) [1000]: " input_eps
+# EXPECTED_EPS=${input_eps:-1000}
+
+# # Calculate optimal settings based on EPS
+# if [ "$EXPECTED_EPS" -gt 10000 ]; then
+#     UDP_SOCKETS=4
+#     PARALLELIZE="yes"
+#     echo -e "${YELLOW}High EPS detected (${EXPECTED_EPS}), enabling performance optimizations${NC}"
+# elif [ "$EXPECTED_EPS" -gt 5000 ]; then
+#     UDP_SOCKETS=3
+#     PARALLELIZE="yes"
+#     echo -e "${YELLOW}Medium-high EPS detected (${EXPECTED_EPS}), using moderate optimizations${NC}"
+# elif [ "$EXPECTED_EPS" -gt 1000 ]; then
+#     UDP_SOCKETS=2
+#     PARALLELIZE="no"
+# fi
+
+# Advanced UDP options
+echo ""
+echo "${GREEN}=== Advanced UDP Options ===${NC}"
+
+# UDP fetch limit overrides
+if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
+    ADJUST_FETCH_LIMIT=$(ask_yes_no "Adjust fetch limit for UDP" "no")
+
+    if [[ "$ADJUST_FETCH_LIMIT" == "yes" ]]; then
+        read -p "UDP fetch limit [$SC4S_SOURCE_UDP_FETCH_LIMIT]: " input_udp_fetch_limit
+        SC4S_SOURCE_UDP_FETCH_LIMIT=${input_udp_fetch_limit:-$SC4S_SOURCE_UDP_FETCH_LIMIT}
+    fi
+fi
+
+# UDP listen socket overrides
+if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
+    ADJUST_LISTEN_SOCKETS=$(ask_yes_no "Adjust number of UDP listen sockets?" "no")
+
+    if [[ "$ADJUST_LISTEN_SOCKETS" == "yes" ]]; then
+        read -p "UDP listen sockets [$SC4S_SOURCE_LISTEN_UDP_SOCKETS]: " input_udp_sockets
+        SC4S_SOURCE_LISTEN_UDP_SOCKETS=${input_udp_sockets:-$SC4S_SOURCE_LISTEN_UDP_SOCKETS}
+    fi
+fi
+
+# UDP receiving buffer overrides
+if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
+    read -p "Tune UDP receiving buffer (-1 to skip, default 17039360 bytes) [$SC4S_SOURCE_UDP_SO_RCVBUFF]: " input_udp_rcvbuff
+    SC4S_SOURCE_UDP_SO_RCVBUFF=${input_udp_rcvbuff:-$SC4S_SOURCE_UDP_SO_RCVBUFF}
+fi
+
+# UDP eBPF options
+if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
+    SC4S_ENABLE_EBPF=$(ask_yes_no "Enable eBPF?" "$SC4S_ENABLE_EBPF")
+    if [[ "$SC4S_ENABLE_EBPF" == "yes" ]]; then
+        read -p "Number of eBPF sockets [4]: " input_ebpf_sockets
+        SC4S_EBPF_NO_SOCKETS=${input_ebpf_sockets:-4}
+    fi
+fi
+
+# Advanced TCP options
+echo ""
+echo "${GREEN}=== Advanced TCP Options ===${NC}"
+
+# TCP receiving buffer overrides
+if [[ "$PROTOCOL" == "tcp" || "$PROTOCOL" == "both" ]]; then
+    read -p "Tune TCP receiving buffer (-1 to skip, default 17039360 bytes) [$SC4S_SOURCE_TCP_SO_RCVBUFF]: " input_tcp_rcvbuff
+    SC4S_SOURCE_TCP_SO_RCVBUFF=${input_tcp_rcvbuff:-$SC4S_SOURCE_TCP_SO_RCVBUFF}
+fi
+
+# TCP parallelization
+if [[ "$PROTOCOL" == "tcp" || "$PROTOCOL" == "both" ]]; then
+    PARALLELIZE=$(ask_yes_no "Enable TCP parallelization?" "$PARALLELIZE")
+    if [[ "$PARALLELIZE" == "yes" ]]; then
+        read -p "Number of partitions for parallelization [4]: " input_partitions
+        SC4S_PARALLELIZE_NO_PARTITION=${input_partitions:-4}
+    fi
+fi
+
+# TCP IW settings
+if [[ "$PROTOCOL" == "tcp" || "$PROTOCOL" == "both" ]]; then
+    SC4S_SOURCE_TCP_IW_USE=$(ask_yes_no "Tune static window size?" "$SC4S_SOURCE_TCP_IW_USE")
+    if [[ "$SC4S_SOURCE_TCP_IW_USE" == "yes" ]]; then
+        read -p "Input window size [1000000]: " input_iw_size
+        SC4S_SOURCE_TCP_IW_SIZE=${input_iw_size:-1000000}
+    fi
+fi
+
+# Disk Buffer Configuration
+echo ""
+echo "${GREEN}=== Disk Buffer Configuration ===${NC}"
+
+ADJUST_DISKBUFF=$(ask_yes_no "Adjust disk buffer settings?" "no")
+
+if [[ "$ADJUST_DISKBUFF" == "yes" ]]; then
+    SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE=$(ask_yes_no "Enable local disk buffering?" "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE")
+
+    if [[ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE" == "yes" ]]; then
+        SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE=$(ask_yes_no "Enable reliable disk buffering (recommended: no for normal buffering)?" "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE")
+
+        if [[ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE" == "yes" ]]; then
+            read -p "Worker memory buffer size in bytes (for reliable buffering) [$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE]: " input_membufsize
+            SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE=${input_membufsize:-$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE}
+        else
+            read -p "Worker memory buffer size in message count (for normal buffering) [$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH]: " input_membuflength
+            SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH=${input_membuflength:-$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH}
+        fi
+
+        read -p "Disk buffer size in bytes (default 50GB per worker) [$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE]: " input_diskbufsize
+        SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE=${input_diskbufsize:-$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE}
+    fi
+fi
+
+# Output file
+echo ""
+read -p "Output filename [$OUTPUT_FILE]: " input_output
+OUTPUT_FILE=${input_output:-$OUTPUT_FILE}
+
+# Generate configuration
+echo ""
+echo "${BLUE}Generating configuration...${NC}"
+
+cat > "$OUTPUT_FILE" << EOF
+# SC4S Configuration - Generated by configuration tool
+# Expected EPS: $EXPECTED_EPS
+# Protocol: $PROTOCOL
+# Generated on: $(date)
+
+# === Splunk HEC Configuration ===
+SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=$SPLUNK_URL
+SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=$HEC_TOKEN
+EOF
+
+if [ "$TLS_VERIFY" == "no" ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no
+EOF
+fi
+
+cat >> "$OUTPUT_FILE" << EOF
+
+# === Performance Configuration ===
+EOF
+
+if [[ "$PROTOCOL" == "udp" || "$PROTOCOL" == "both" ]]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_SOURCE_UDP_FETCH_LIMIT=$SC4S_SOURCE_UDP_FETCH_LIMIT
+SC4S_SOURCE_LISTEN_UDP_SOCKETS=$SC4S_SOURCE_LISTEN_UDP_SOCKETS
+EOF
+fi
+
+if [ "$SC4S_SOURCE_UDP_SO_RCVBUFF" -gt 0 ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_SOURCE_UDP_SO_RCVBUFF=$SC4S_SOURCE_UDP_SO_RCVBUFF
+EOF
+fi
+
+if [ "$SC4S_ENABLE_EBPF" == "yes" ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_ENABLE_EBPF=$SC4S_ENABLE_EBPF
+SC4S_EBPF_NO_SOCKETS=$SC4S_EBPF_NO_SOCKETS
+EOF
+fi
+
+cat >> "$OUTPUT_FILE" << EOF
+
+EOF
+
+if [ "$SC4S_SOURCE_TCP_SO_RCVBUFF" -gt 0 ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_SOURCE_TCP_SO_RCVBUFF=$SC4S_SOURCE_TCP_SO_RCVBUFF
+EOF
+fi
+
+if [ "$PARALLELIZE" == "yes" ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_ENABLE_PARALLELIZE=yes
+SC4S_PARALLELIZE_NO_PARTITION=$SC4S_PARALLELIZE_NO_PARTITION
+EOF
+fi
+
+if [ "$SC4S_SOURCE_TCP_IW_USE" == "yes" ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_SOURCE_TCP_IW_SIZE=$SC4S_SOURCE_TCP_IW_SIZE
+EOF
+fi
+
+# === Disk Buffer Configuration ===
+if [ "$ADJUST_DISKBUFF" == "yes" ]; then
+    cat >> "$OUTPUT_FILE" << EOF
+
+# === Disk buffer Configuration ===
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE
+EOF
+
+    if [ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE" == "yes" ]; then
+        cat >> "$OUTPUT_FILE" << EOF
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE
+EOF
+
+    if [ "$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_RELIABLE" == "yes" ]; then
+        cat >> "$OUTPUT_FILE" << EOF
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFSIZE
+EOF
+    else
+        cat >> "$OUTPUT_FILE" << EOF
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_MEMBUFLENGTH
+EOF
+    fi
+
+    cat >> "$OUTPUT_FILE" << EOF
+SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE=$SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_DISKBUFSIZE
+EOF
+fi
+fi
+
+echo ""
+echo "${GREEN}✓ Configuration generated successfully!${NC}"
+echo "File: ${YELLOW}$OUTPUT_FILE${NC}"
+echo ""
+echo "${BLUE}Configuration Summary:${NC}"
+echo "  Splunk URL: $SPLUNK_URL"
+echo "  Protocol: $PROTOCOL"
+echo "  Expected EPS: $EXPECTED_EPS"
+echo ""
+echo "${GREEN}Configuration tool completed successfully!${NC}"
+
+
+# === Final recommendations ===
+if [[ "$SC4S_SOURCE_UDP_SO_RCVBUFF" -gt 0 || "$SC4S_SOURCE_TCP_SO_RCVBUFF" -gt 0 ]]; then
+echo ""
+echo "${YELLOW}Note: You may need to adjust your system's UDP/TCP receiving buffer settings to match the configured values.${NC}"
+echo "You can modify /etc/sysctl.conf following this documentation:"
+echo "https://splunk.github.io/splunk-connect-for-syslog/main/gettingstarted/getting-started-runtime-configuration/#tune-your-receive-buffer"
+fi
+
+
+if [ "$SC4S_ENABLE_EBPF" == "yes" ]; then
+echo ""
+echo "${YELLOW}Note: Enabling eBPF may require additional system permissions.${NC}"
+echo "Ensure that your system supports eBPF and that the necessary capabilities are granted to the SC4S process or container. Read more here: "
+echo "https://splunk.github.io/splunk-connect-for-syslog/main/configuration/#about-ebpf"
+fi

docs/architecture/index.md

@@ -54,3 +54,25 @@ The following resources can help you determine the best protocol for your setup:
 
 1. [Run performance tests for TCP](performance-tests.md#check-your-tcp-performance)
 2. [Run performance tests for UDP](performance-tests.md#check-your-udp-performance)
+
+### Configuration helper for SC4S
+
+SC4S includes an interactive configuration tool that simplifies the setup and optimization of your environment file. The tool walks you through key configuration decisions and generates a customized `env_file` based on your specific requirements.
+
+**What the tool configures:**
+
+* **Splunk HEC destination** - Connection URL, token, and TLS verification settings
+* **Protocol optimization** - Choose UDP, TCP, or both, with protocol-specific tuning options
+* **UDP settings** - Socket configuration, fetch limits, receive buffers, and eBPF support for high-throughput scenarios
+* **TCP settings** - Receive buffers, parallelization, and static window size tuning
+* **Disk buffering** - Local disk buffer settings to minimize data loss during connectivity issues
+
+**To use the configuration tool:**
+
+```bash
+sh ./configuration-tool.sh
+```
+
+The tool will prompt you for your environment details and create an optimized `env_file` ready for deployment. This approach ensures consistent configuration and helps you apply best practices without manually editing environment variables.
+
+You can also read about our [scaling solutions](performance-tests.md) and [configuration variables](../configuration.md).

mkdocs.yml

@@ -77,6 +77,7 @@ nav:
   - SC4S Lite:
       - Intro: "lite.md"
       - Pluggable modules: "pluggable_modules.md"
+  - Enterprise: enterprise.md
   - Edge Processor: "edge_processor.md"
   - Troubleshooting:
       - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"