monitoring.agent

#!/bin/sh
#
# OpenWISP Monitoring Daemon

show_help() {
	printf "Usage:\n"
	printf "  openwisp-monitoring [OPTIONS...]\n"
	printf "\n"
	printf "Runs OpenWISP Monitoring Agent for collecting and sending data.\n"
	printf "\n"
	printf "General options:\n"
	printf "  -h, --help\t\t\t\t: Show this help text\n"
	printf "  -v, --version\t\t\t\t: Shows version of the agent.\n"
	printf "  --mode <send, collect>\t\t: Mode for running Monitoring Agent.\n"
	printf "OpenWISP Config options:\n"
	printf "  --url <server-url>\t\t\t: URL of the OpenWISP server.\n"
	printf "  --uuid <uuid>\t\t\t\t: UUID of the device.\n"
	printf "  --key <key>\t\t\t\t: Key for the device.\n"
	printf "  --verify_ssl <0, 1>\t\t\t: Whether SSL Authentication should be enabled or not.\n"
	printf "  --cacert <file>\t\t\t: Value passed to curl --cacert argument.\n"
	printf "  --capath <directory>\t\t\t: Value passed to curl --capath argument.\n"
	printf "OpenWISP Monitoring Config options:\n"
	printf "  --interval <time>\t\t\t: Time in seconds after which data should be sent/collected.\n"
	printf "  --monitored_interfaces <interfaces>\t: Interfaces that needs to be monitored.\n"
	printf "  --verbose_mode <0, 1>\t\t\t: Run agent in verbose mode.\n"
	printf "  --required_memory <0-1>\t\t: Fraction of total memory that should be available to collect data.\n"
	printf "  --max_retries <max-retries-count>\t: No. of time agent should retry to send data.\n"
	exit 0
}

show_version() {
	VERSION=$(cat /usr/lib/openwisp-monitoring/VERSION)
	echo "$(basename "$0") $VERSION"
	exit 0
}

echoerr() { echo "$@" 1>&2 && exit 1; }

log() {
	level="$1"
	type="$2"
	shift 2

	if [ "$type" = "-v" ] && [ "$VERBOSE_MODE" -ne "1" ]; then
		return 0
	fi

	case "$level" in
		-i) level=daemon.info ;;
		-w) level=daemon.warn ;;
		-e) level=daemon.err ;;
		-*)
			echoerr "Invalid message level : $level"
			;;
	esac

	logger -s "$@" -p "$level" -t openwisp-monitoring
}

time_to_seconds() {
	time=$1

	{ [ "$time" -ge 1 ] 2>/dev/null && seconds="$time"; } \
		|| { [ "${time%s}" -ge 1 ] 2>/dev/null && seconds="${time%s}"; } \
		|| { [ "${time%m}" -ge 1 ] 2>/dev/null && seconds=$((${time%m} * 60)); } \
		|| { [ "${time%h}" -ge 1 ] 2>/dev/null && seconds=$((${time%h} * 3600)); } \
		|| { [ "${time%d}" -ge 1 ] 2>/dev/null && seconds=$((${time%d} * 86400)); }

	echo "$seconds"
	unset seconds
	unset time
}

check_available_memory() {
	while true; do
		total=$(ubus call system info | jsonfilter -e '@.memory.total')
		available=$(ubus call system info | jsonfilter -e '@.memory.available')
		required=$(echo - | awk -v percent="$REQUIRED_PERCENT" -v total="$total" '{printf("%.f",percent*total)}')

		if [ "$available" -ge "$required" ]; then
			return 0
		else
			# enough memory not available, deleting old data
			# shellcheck disable=SC2012
			file="$TMP_DIR/$(ls -1t "$TMP_DIR" | tail -1)"
			# ensure that memory was available previously and file was written
			if [ -f "$file" ]; then
				rm "$file"
			else
				log -w -n "Not enough memory available, skipping collect data."
				return 1
			fi
		fi
	done
}

collect_data() {
	n=0
	log -i -v "Collecting NetJSON Monitoring data."
	until [ "$n" -ge 5 ]; do
		/usr/sbin/netjson-monitoring --dump "$MONITORED_INTERFACES" && break

		[ "$n" -eq 4 ] && log -e -v "Collecting data failed!"
		n=$((n + 1))
		sleep 5
	done
}

set_url_and_curl() {
	[ -z "$BASE_URL" ] && echoerr "missing required --url option"

	[ -z "$UUID" ] && echoerr "missing required --uuid option"

	[ -z "$KEY" ] && echoerr "missing required --key option"

	URL="$BASE_URL/api/v1/monitoring/device/$UUID/?key=$KEY"

	# shellcheck disable=SC1083
	CURL_COMMAND="curl -s -w "%{http_code}" --output $RESPONSE_FILE"
	[ "$VERIFY_SSL" -eq "0" ] && CURL_COMMAND="$CURL_COMMAND -k"
	[ -n "$CACERT" ] && CURL_COMMAND="$CURL_COMMAND --cacert $CACERT"
	[ -n "$CAPATH" ] && CURL_COMMAND="$CURL_COMMAND --capath $CAPATH"
	[ "$VERBOSE_MODE" -eq "1" ] && CURL_COMMAND="$CURL_COMMAND -v"
	MAX_RETRIES=${MAX_RETRIES:-5}
	FAILING=0
	return 0
}

save_data() {
	while true; do
		if check_available_memory; then
			data="$(collect_data)"
			filename="$(date -u +'%d-%m-%Y_%H:%M:%S')"
			# save data with file_name
			echo "$data" >"$TMP_DIR/$filename"
			# compress data
			gzip "$TMP_DIR/$filename"
			log -i -v "Data saved temporarily."
		fi
		# get process id of the process sending data
		pid=$(pgrep -f "openwisp-monitoring.*--mode send")
		kill -SIGUSR1 "$pid"
		sleep "$INTERVAL"
	done
}

handle_sigusr1() {
	log -i -v "SIGUSR1 received! Sending data."
	return 0
}

send_data() {
	success=0
	while true; do
		for file in "$TMP_DIR"/*; do
			if [ ! -f "$file" ]; then
				log -i -v "No data file found to send. Checking again in $INTERVAL seconds."
				trap handle_sigusr1 USR1
				# SIGUSR1 signal received, interrupt sleep and continue sending data
				sleep "$INTERVAL" &
				wait $!
				continue
			fi
			trap "" USR1
			basefilename=${file##*/}
			filename=${basefilename%.*}
			# remove previous saved response if exist
			[ "$filename" = "response" ] && rm "$file" 2>/dev/null && continue
			# extra zeroes are added for nanoseconds precision
			url="$URL&time=$filename.000000"
			# retry sending data in case of failure
			failures=0
			# check if the data is latest or old one
			[ "$(echo "$TMP_DIR"/* | awk '{print $2}')" ] || url="$url&current=true"
			if [ "${basefilename##*.}" = "gz" ]; then
				# decompress file
				gzip -d "$file"
			fi
			filename="$TMP_DIR/$filename"
			# check if the data file exist
			if [ -f "$filename" ]; then
				data=$(cat "$filename")
			else
				log -i -v "data file $filename not found."
				continue
			fi
			while true; do
				if [ "$failures" -eq "$MAX_RETRIES" ]; then
					[ -f "$RESPONSE_FILE" ] && error_message="\"$(cat "$RESPONSE_FILE")\"" || error_message='"".'
					log -e -v "Data not sent successfully. Response code is \"$response_code\"." \
						"Error message is $error_message"
					# check if agent was already passing or not to avoid repeating log messages
					if [ "$FAILING" -eq "0" ]; then
						FAILING=1
						[ "$VERBOSE_MODE" -ne "1" ] && log -e -n "Data not sent successfully. Response code is \"$response_code\"." \
							"Run with verbose mode to find more."
					fi
					break
				fi
				# send data
				response_code=$($CURL_COMMAND -H "Content-Type: application/json" -d "$data" "$url")
				if [ "$response_code" = "200" ]; then
					success=$((success + 1))
					log -i -v "Data sent successfully."
					# check if agent was already failing or not to avoid repeating log messages
					if [ "$FAILING" -eq "1" ]; then
						FAILING=0
						rm -f "$RESPONSE_FILE"
						[ "$VERBOSE_MODE" -ne "1" ] && log -i -n "Data sent successfully."
					fi
					# remove saved data
					rm -f "$filename"
					break
				elif [ "$response_code" = "400" ]; then
					log -i -n "Data not sent successfully (HTTP status $response_code), discarding data."
					rm -f "$filename"
					break
				else
					timeout=$(/usr/sbin/openwisp-get-random-number 2 15)
					log -w -v "Data not sent successfully. Retrying in $timeout seconds."
					failures=$((failures + 1))
					if [ "$response_code" = "404" ]; then
						# If we get a HTTP 404 response, it could mean that the device has been deleted from OpenWISP
						# Controller. We check if openwisp-config agent is running to determine if the device has been
						# deleted. If openwisp-config agent is not running, the monitoring agent will also exit.
						if ! pgrep -x "openwisp-config" >/dev/null; then
							log -e -n "Giving up and shutting down: the device may have been deleted from OpenWISP Controller"
							# get process id of the process collecting data
							pid=$(pgrep -f "openwisp-monitoring.*--mode collect")
							kill -SIGKILL "$pid"
							exit 2
						fi
					fi
					sleep "$timeout"
				fi
			done
			# retry sending same data again in next cycle
			[ "$failures" -eq "$MAX_RETRIES" ] && break
			# pause for a random time between 1 and 5 seconds every
			# 10 successful requests sent to avoid overload the server
			if [ $((success % 10)) -eq 0 ]; then
				pause_duration=$(/usr/sbin/openwisp-get-random-number 1 5)
				sleep "$pause_duration"
			fi
		done
	done
}

wait_until_registered() {
	if [ -n "$UUID" ] && [ -n "$KEY" ]; then
		return 0
	fi
	log -i -n "Waiting for device to register."
	UUID=$(uci get openwisp.http.uuid 2>/dev/null)
	KEY=$(uci get openwisp.http.key 2>/dev/null)
	if [ -z "$UUID" ] || [ -z "$KEY" ]; then
		return 1
	fi
	log -i -n "Setting uuid and key."
	export UUID KEY
}

bootup_delay() {
	if [ ! -f "$BOOTUP" ]; then
		# actions to do only after agent start, not after restart
		if [ "$BOOTUP_DELAY" -ne "0" ]; then
			# get a random number between zero and $BOOTUP_DELAY
			DELAY=$(/usr/sbin/openwisp-get-random-number 0 "$BOOTUP_DELAY")
			log -i -n "Delaying initialization of the monitoring agent for $DELAY seconds."
			sleep "$DELAY"
		fi
		# send bootup hotplug event
		touch "$BOOTUP"
	fi
}

main() {
	# parse options
	while [ -n "$1" ]; do
		case "$1" in
			--version | -v)
				show_version
				;;
			--help | -h)
				show_help
				;;
			--url)
				export BASE_URL="$2"
				shift
				;;
			--uuid)
				export UUID="$2"
				shift
				;;
			--key)
				export KEY="$2"
				shift
				;;
			--verify_ssl)
				export VERIFY_SSL="$2"
				shift
				;;
			--capath)
				export CAPATH="$2"
				shift
				;;
			--cacert)
				export CACERT="$2"
				shift
				;;
			--interval)
				export INTERVAL="$2"
				shift
				;;
			--monitored_interfaces)
				export MONITORED_INTERFACES="$2"
				shift
				;;
			--verbose_mode)
				export VERBOSE_MODE="$2"
				shift
				;;
			--required_memory)
				export REQUIRED_PERCENT="$2"
				shift
				;;
			--mode)
				export MODE="$2"
				shift
				;;
			--max_retries)
				export MAX_RETRY="$2"
				shift
				;;
			--bootup_delay)
				export BOOTUP_DELAY="$2"
				shift
				;;
			-*)
				echoerr "Invalid option: $1."
				;;
			*) break ;;
		esac
		shift
	done

	INTERVAL=${INTERVAL:-300}
	INTERVAL="$(time_to_seconds "$INTERVAL")"
	[ -z "$INTERVAL" ] && echoerr "Interval is invalid. Use time value(eg: '10', '2m', '3h', '1d')"
	REGISTRATION_INTERVAL=$((INTERVAL / 10))
	VERBOSE_MODE=${VERBOSE_MODE:-0}
	BOOTUP_DELAY=${BOOTUP_DELAY:-10}
	TMP_DIR="/tmp/openwisp/monitoring"
	# The agent tries to upload all files in /tmp/openwisp/monitoring
	# to the controller, hence /tmp/openwisp/monitoring-bootup
	# is used to prevent agent from uploading this file.
	BOOTUP="$TMP_DIR-bootup"

	[ -z "$MODE" ] && echoerr "missing required --mode option"

	# make directory
	mkdir -p "$TMP_DIR"

	if [ "$MODE" = "collect" ]; then
		bootup_delay
		MONITORED_INTERFACES=${MONITORED_INTERFACES:-*}
		# remove double quotes from interfaces
		MONITORED_INTERFACES=$(echo "$MONITORED_INTERFACES" | tr -d '"')
		save_data
	elif [ "$MODE" = "send" ]; then
		until wait_until_registered; do
			sleep "$REGISTRATION_INTERVAL"
		done
		VERIFY_SSL=${VERIFY_SSL:-0}
		RESPONSE_FILE="$TMP_DIR"/response.txt
		set_url_and_curl && send_data
	else
		echoerr "The supplied mode is invalid. Only send and collect are allowed."
	fi
}

main "$@"