monitoring.agent

#!/bin/sh
#
# OpenWISP Monitoring Daemon

show_help() {
	printf "Usage:\n"
	printf "  openwisp-monitoring [OPTIONS...]\n"
	printf "\n"
	printf "Runs OpenWISP Monitoring Agent for collecting and sending data.\n"
	printf "\n"
	printf "General options:\n"
	printf "  -h, --help\t\t\t\t: Show this help text\n"
	printf "  -v, --version\t\t\t\t: Shows version of the agent.\n"
	printf "  --mode <send, collect>\t\t: Mode for running Monitoring Agent.\n"
	printf "OpenWISP Config options:\n"
	printf "  --url <server-url>\t\t\t: URL of the OpenWISP server.\n"
	printf "  --uuid <uuid>\t\t\t\t: UUID of the device.\n"
	printf "  --key <key>\t\t\t\t: Key for the device.\n"
	printf "  --verify_ssl <0, 1>\t\t\t: Whether SSL Authentication should be enabled or not.\n"
	printf "  --cacert <file>\t\t\t: Value passed to curl --cacert argument.\n"
	printf "  --capath <directory>\t\t\t: Value passed to curl --capath argument.\n"
	printf "OpenWISP Monitoring Config options:\n"
	printf "  --interval <time>\t\t\t: Time in seconds after which data should be sent/collected.\n"
	printf "  --monitored_interfaces <interfaces>\t: Interfaces that needs to be monitored.\n"
	printf "  --verbose_mode <0, 1>\t\t\t: Run agent in verbose mode.\n"
	printf "  --required_memory <0-1>\t\t: Fraction of total memory that should be available to collect data.\n"
	printf "  --max_retries <max-retries-count>\t: No. of time agent should retry to send data.\n"
	exit 0
}

show_version() {
	VERSION=$(cat /usr/lib/openwisp-monitoring/VERSION)
	echo "$(basename "$0") $VERSION"
	exit 0
}

echoerr() { echo "$@" 1>&2 && exit 1; }

check_available_memory() {
	while true; do
		total=$(ubus call system info | jsonfilter -e '@.memory.total')
		available=$(ubus call system info | jsonfilter -e '@.memory.available')
		required=$(echo - | awk -v percent="$REQUIRED_PERCENT" -v total="$total" '{printf("%.f",percent*total)}')

		if [ "$available" -ge "$required" ]; then
			return 0
		else
			# enough memory not available, deleting old data
			# shellcheck disable=SC2012
			file="$TMP_DIR/$(ls -1t "$TMP_DIR" | tail -1)"
			# ensure that memory was available previously and file was written
			if [ -f "$file" ]; then
				rm "$file"
			else
				[ "$VERBOSE_MODE" -eq "1" ] \
					&& logger -s "Not enough memory available, skipping collect data." \
						-p daemon.warn
				return 1
			fi
		fi
	done
}

collect_data() {
	n=0
	[ "$VERBOSE_MODE" -eq "1" ] && logger -s "Collecting NetJSON Monitoring data." \
		-p daemon.info
	until [ "$n" -ge 5 ]; do
		/usr/sbin/netjson-monitoring --dump "$MONITORED_INTERFACES" && break

		if [ "$n" -eq 5 ]; then
			[ "$VERBOSE_MODE" -eq "1" ] && logger -s "Collecting data failed!" -p daemon.err
		fi
		n=$((n + 1))
		sleep 5
	done
}

set_url_and_curl() {
	[ -z "$BASE_URL" ] && echoerr "missing required --url option"

	[ -z "$UUID" ] && echoerr "missing required --uuid option"

	[ -z "$KEY" ] && echoerr "missing required --key option"

	URL="$BASE_URL/api/v1/monitoring/device/$UUID/?key=$KEY"

	# shellcheck disable=SC1083
	CURL_COMMAND="curl -s -w "%{http_code}" --output $RESPONSE_FILE"
	[ "$VERIFY_SSL" -eq "0" ] && CURL_COMMAND="$CURL_COMMAND -k"
	[ -n "$CACERT" ] && CURL_COMMAND="$CURL_COMMAND --cacert $CACERT"
	[ -n "$CAPATH" ] && CURL_COMMAND="$CURL_COMMAND --capath $CAPATH"
	[ "$VERBOSE_MODE" -eq "1" ] && CURL_COMMAND="$CURL_COMMAND -v"
	MAX_RETRIES=${MAX_RETRIES:-5}
	FAILING=0
	return 0
}

save_data() {
	while true; do
		if check_available_memory; then
			data="$(collect_data)"
			filename="$(date -u +'%d-%m-%Y_%H:%M:%S')"
			# save data with file_name
			echo "$data" >"$TMP_DIR/$filename"
			# compress data
			gzip "$TMP_DIR/$filename"
			[ "$VERBOSE_MODE" -eq "1" ] && logger -s "Data saved temporarily." \
				-p daemon.info
		fi
		# get process id of the process sending data
		pid=$(pgrep -f "openwisp-monitoring.*--mode send")
		kill -SIGUSR1 "$pid"
		sleep "$INTERVAL"
	done
}

handle_sigusr1() {
	[ "$VERBOSE_MODE" -eq "1" ] && logger -s "SIGUSR1 received! Sending data." \
		-p daemon.info
	return 0
}

send_data() {
	success=0
	while true; do
		for file in "$TMP_DIR"/*; do
			if [ ! -f "$file" ]; then
				[ "$VERBOSE_MODE" -eq "1" ] && logger -s "No data file found to send." \
					-p daemon.info
				trap handle_sigusr1 USR1
				# SIGUSR1 signal received, interrupt sleep and continue sending data
				sleep "$INTERVAL" &
				wait $!
				continue
			fi
			trap "" USR1
			basefilename=${file##*/}
			filename=${basefilename%.*}
			# remove previous saved response if exist
			[ "$filename" = "response" ] && rm "$file" 2>/dev/null && continue
			# extra zeroes are added for nanoseconds precision
			url="$URL&time=$filename.000000"
			# retry sending data in case of failure
			failures=0
			# check if the data is latest or old one
			[ "$(echo "$TMP_DIR"/* | awk '{print $2}')" ] || url="$url&current=true"
			if [ "${basefilename##*.}" = "gz" ]; then
				# decompress file
				gzip -d "$file"
			fi
			filename="$TMP_DIR/$filename"
			# check if the data file exist
			if ! [ -f "$filename" ]; then
				[ "$VERBOSE_MODE" -eq "1" ] && logger -s "data file $filename not found." \
					-p daemon.info
				continue
			fi
			while true; do
				if [ "$failures" -eq "$MAX_RETRIES" ]; then
					[ -f "$RESPONSE_FILE" ] && error_message="\"$(cat "$RESPONSE_FILE")\"" || error_message='"".'
					if [ "$VERBOSE_MODE" -eq "1" ]; then
						logger -s "Data not sent successfully. Response code is \"$response_code\"." \
							"Error message is $error_message" \
							-p daemon.err
					elif [ "$FAILING" -eq "0" ]; then
						FAILING=1
						logger -s "Data not sent successfully. Response code is \"$response_code\"." \
							"Error message is $error_message" \
							"Run with verbose mode to find more." \
							-t openwisp-monitoring \
							-p daemon.err
					fi
					break
				fi
				# send data
				if [ "$VERBOSE_MODE" -eq "1" ]; then
					printf "data="
					cat "$filename"
					set -x
				fi
				response_code=$($CURL_COMMAND -H "Content-Type: application/json" -d "@$filename" "$url")
				set +x
				if [ "$response_code" = "200" ]; then
					success=$((success + 1))
					if [ "$VERBOSE_MODE" -eq "1" ]; then
						logger -s "Data sent successfully." \
							-p daemon.info
					elif [ "$FAILING" -eq "1" ]; then
						logger -s "Data sent successfully." \
							-t openwisp-monitoring \
							-p daemon.info
						FAILING=0
						rm -f "$RESPONSE_FILE"
					fi
					# remove saved data
					rm -f "$filename"
					break
				elif [ "$response_code" = "400" ]; then
					logger -s "Data not sent successfully (HTTP status $response_code), discarding data." \
						-t openwisp-monitoring \
						-p daemon.err
					rm -f "$filename"
					break
				else
					timeout=$(/usr/sbin/openwisp-get-random-number 2 15)
					[ "$VERBOSE_MODE" -eq "1" ] && logger -s "Data not sent successfully. Retrying in $timeout seconds." \
						-p daemon.warn
					failures=$((failures + 1))
					if [ "$response_code" = "404" ]; then
						# If we get a HTTP 404 response, it could mean that the device has been deleted from OpenWISP
						# Controller. We check if openwisp-config agent is running to determine if the device has been
						# deleted. If openwisp-config agent is not running, the monitoring agent will also exit.
						if ! pgrep -x "openwisp-config" >/dev/null; then
							logger -s "Giving up and shutting down: the device may have been deleted from OpenWISP Controller" \
								-t openwisp-monitoring \
								-p daemon.err
							# get process id of the process collecting data
							pid=$(pgrep -f "openwisp-monitoring.*--mode collect")
							kill -SIGKILL "$pid"
							exit 2
						fi
					fi
					sleep "$timeout"
				fi
			done
			# retry sending same data again in next cycle
			[ "$failures" -eq "$MAX_RETRIES" ] && break
			# pause for a random time between 1 and 5 seconds every
			# 10 successful requests sent to avoid overload the server
			if [ $((success % 10)) -eq 0 ]; then
				pause_duration=$(/usr/sbin/openwisp-get-random-number 1 5)
				sleep "$pause_duration"
			fi
		done
	done
}

wait_until_registered() {
	if [ -n "$UUID" ] && [ -n "$KEY" ]; then
		return 0
	fi
	logger -s "Waiting for device to register." \
		-t openwisp-monitoring \
		-p daemon.info
	UUID=$(uci get openwisp.http.uuid 2>/dev/null)
	KEY=$(uci get openwisp.http.key 2>/dev/null)
	if [ -z "$UUID" ] || [ -z "$KEY" ]; then
		return 1
	fi
	logger -s "Setting uuid and key." \
		-t openwisp-monitoring \
		-p daemon.info
	export UUID KEY
}

bootup_delay() {
	if [ ! -f "$BOOTUP" ]; then
		# actions to do only after agent start, not after restart
		if [ "$BOOTUP_DELAY" -ne "0" ]; then
			# get a random number between zero and $BOOTUP_DELAY
			DELAY=$(/usr/sbin/openwisp-get-random-number 0 "$BOOTUP_DELAY")
			logger "Delaying initialization of the monitoring agent for $DELAY seconds." \
				-t openwisp-monitoring \
				-p daemon.info
			sleep "$DELAY"
		fi
		# send bootup hotplug event
		touch "$BOOTUP"
	fi
}

main() {
	# parse options
	while [ -n "$1" ]; do
		case "$1" in
			--version | -v)
				show_version
				;;
			--help | -h)
				show_help
				;;
			--url)
				export BASE_URL="$2"
				shift
				;;
			--uuid)
				export UUID="$2"
				shift
				;;
			--key)
				export KEY="$2"
				shift
				;;
			--verify_ssl)
				export VERIFY_SSL="$2"
				shift
				;;
			--capath)
				export CAPATH="$2"
				shift
				;;
			--cacert)
				export CACERT="$2"
				shift
				;;
			--interval)
				export INTERVAL="$2"
				shift
				;;
			--monitored_interfaces)
				export MONITORED_INTERFACES="$2"
				shift
				;;
			--verbose_mode)
				export VERBOSE_MODE="$2"
				shift
				;;
			--required_memory)
				export REQUIRED_PERCENT="$2"
				shift
				;;
			--mode)
				export MODE="$2"
				shift
				;;
			--max_retries)
				export MAX_RETRY="$2"
				shift
				;;
			--bootup_delay)
				export BOOTUP_DELAY="$2"
				shift
				;;
			-*)
				echo "Invalid option: $1"
				exit 1
				;;
			*) break ;;
		esac
		shift
	done

	INTERVAL=${INTERVAL:-300}
	REGISTRATION_INTERVAL=$((INTERVAL / 10))
	VERBOSE_MODE=${VERBOSE_MODE:-0}
	BOOTUP_DELAY=${BOOTUP_DELAY:-10}
	TMP_DIR="/tmp/openwisp/monitoring"
	# The agent tries to upload all files in /tmp/openwisp/monitoring
	# to the controller, hence /tmp/openwisp/monitoring-bootup
	# is used to prevent agent from uploading this file.
	BOOTUP="$TMP_DIR-bootup"

	[ -z "$MODE" ] && echoerr "missing required --mode option"

	# make directory
	mkdir -p "$TMP_DIR"

	if [ "$MODE" = "collect" ]; then
		bootup_delay
		MONITORED_INTERFACES=${MONITORED_INTERFACES:-*}
		# remove double quotes from interfaces
		MONITORED_INTERFACES=$(echo "$MONITORED_INTERFACES" | tr -d '"')
		save_data
	elif [ "$MODE" = "send" ]; then
		until wait_until_registered; do
			sleep "$REGISTRATION_INTERVAL"
		done
		VERIFY_SSL=${VERIFY_SSL:-0}
		RESPONSE_FILE="$TMP_DIR"/response.txt
		set_url_and_curl && send_data
	else
		echoerr "The supplied mode is invalid. Only send and collect are allowed"
	fi
}

main "$@"