[feature] Resilient sending: store data for later sending if offline #29

devkapilbansal · nemesifier · commit b55b3e2daaef · 2021-08-22T14:08:51.000-05:00
Closes #29
diff --git a/README.rst b/README.rst
@@ -40,6 +40,48 @@ UCI configuration options must go in ``/etc/config/monitoring``.
 - ``monitored_interfaces``: interfaces that needs to be monitored, defaults to ``*`` for all interfaces.
 - ``interval``: time after which device data should be sent to server, defaults to ``300``.
 - ``verbose_mode``: can be enabled (set to ``1``) to ease `debugging <#debugging>`__ in case of issues, defaults to ``0`` (disabled).
+- ``required_memory``: available memory required to save data temporarily, defaults to ``0.05`` (5 percent).
+- ``max_retries``: maximum number of retries in case of failures to send data to server in case of failure, defaults to ``5`` retries.
+
+In case, `maximum retries are reached <#send-mode>`_, agent will try sending data again in next cycle.
+
+Collecting vs Sending
+---------------------
+
+We use two procd services in `monitoring agent <https://github.com/openwisp/openwrt-openwisp-monitoring/blob/master/openwrt-openwisp-monitoring/files/monitoring.agent>`_, one for collecting the data and other for sending the data.
+
+This helps handle failure in sending the data in more flexible way. Old data saved during network connectivity issues can be sent while new data is being collected. If old data has piled up and takes several minutes to be uploaded, new data will be collected without waiting for the sending to complete.
+
+Monitoring agent uses two different modes to handle this, ``send`` and ``collect``.
+
+Collect Mode
+~~~~~~~~~~~~
+
+If openwisp_monitoring agent is called with this mode, then the agent will keep charge of collecting and saving data.
+
+Agent will periodically check if enough memory is available. If true, data will be collected and saved in temporary storage with the timestamp (in UTC timezone).
+
+Once the data is saved, a signal will be sent to the other agent to ensure data is sent as soon as it is collected.
+
+**Note:** Date and time on device should be set correctly. Otherwise, data will be saved with wrong timestamp in timeseries database.
+
+Send Mode
+~~~~~~~~~
+
+If openwisp_monitoring agent is called with this mode, then the agent will keep charge of sending data.
+
+Agent will check if any data file is available in temporary storage.
+
+If there is no data file, the agent will sleep for the time interval and check for the data file again. This will be continued until a data file is found.
+If a signal is received from the other agent, then the sleep will be interrupted and agent will start sending data.
+
+If agent fails to send data to the server, an exponential backoff will be used to retry until `max_retries` is reached.
+If all attempts of sending data failed, the agent will try to send data in the next cycle.
+
+If data is sent successfully, then the data file will be deleted and agent will look for another file.
+
+**SIGUSR1** signals are used to instantly send the data when collected. However, the service will keep trying
+to send data periodically.
 
 Compiling openwrt-openwisp-monitoring
 -------------------------------------
@@ -96,7 +138,6 @@ you will need to select the *openwisp-monitoring* variant and *netjson-monitorin
     ./scripts/feeds update -a
     ./scripts/feeds install -a
     make menuconfig
-    # go to Base system, then select rpcd
     # go to Administration > admin > openwisp and select the packages you need interactively
     make tools/install
     make toolchain/install
@@ -124,6 +165,7 @@ If you are in that doubt openwisp-monitoring is running at all or not, you can c
 
 You should see something like::
 
+    2712 root      1224 S    /bin/sh /usr/sbin/openwisp_monitoring --interval 300 --monitored_interfaces ...
     2713 root      1224 S    /bin/sh /usr/sbin/openwisp_monitoring --url http://192.168.1.195:8000 ...
 
 You can inspect the version of openwisp-monitoring currently installed with::
diff --git a/openwrt-openwisp-monitoring/files/monitoring.agent b/openwrt-openwisp-monitoring/files/monitoring.agent
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-VERSION=0 #default value of version
+VERSION=0 # default value of version 
 while [ -n "$1" ]; do
   case "$1" in
     --version|-v) export VERSION=1; break;;
@@ -10,7 +10,10 @@ while [ -n "$1" ]; do
     --verify_ssl) export VERIFY_SSL="$2"; shift;;
     --interval) export INTERVAL="$2"; shift;;
     --monitored_interfaces) export MONITORED_INTERFACES="$2"; shift;;
-    --verbose_mode) export verbose_mode=$2; shift;;
+    --verbose_mode) export VERBOSE_MODE="$2"; shift;;
+    --required_memory) export REQUIRED_PERCENT="$2"; shift;;
+    --mode) export MODE="$2"; shift;;
+    --max_retries) export MAX_RETRY="$2"; shift;;
     -*)
       echo "Invalid option: $1"
       exit 1
@@ -26,76 +29,178 @@ if [ "$VERSION" -eq "1" ]; then
   exit 0
 fi
 
-VERIFY_SSL=${VERIFY_SSL:-0}
 INTERVAL=${INTERVAL:-300}
-MONITORED_INTERFACES=${MONITORED_INTERFACES:-*}
-verbose_mode=${verbose_mode:-0}
+VERBOSE_MODE=${VERBOSE_MODE:-0}
+TMP_DIR="/tmp/openwisp/monitoring"
 
-if [ -z "$BASE_URL" ]; then
-  logger -s "missing required --url option" \
-         -t openwisp_monitoring \
-         -p daemon.err
-  exit 1
-fi
+echoerr() { echo "$@" 1>&2; }
 
-if [ -z "$UUID" ]; then
-  logger -s "missing required --uuid option" \
-          -t openwisp_monitoring \
-          -p daemon.err
-  exit 1
-fi
+check_available_memory(){
+  total=$(ubus call system info | jsonfilter -e '@.memory.total')
+  available=$(ubus call system info | jsonfilter -e '@.memory.available')
+  required=$(echo - |awk -v percent="$REQUIRED_PERCENT" -v total="$total" '{printf("%.f",percent*total)}')
 
-if [ -z "$KEY" ]; then
-  logger -s "missing required --key option" \
-         -t openwisp_monitoring \
-         -p daemon.err
-  exit 1
-fi
-
-# Remove double quotes from interfaces
-MONITORED_INTERFACES=$(echo "$MONITORED_INTERFACES" | tr -d '"')
-
-URL="$BASE_URL/api/v1/monitoring/device/$UUID/?key=$KEY"
-
-CURL_COMMAND="curl -s -w "%{http_code}""
-[ "$VERIFY_SSL" = "0" ] && CURL_COMMAND="$CURL_COMMAND -k"
-[ "$verbose_mode" = "1" ] && CURL_COMMAND="$CURL_COMMAND -v"
+  if [ "$available" -ge "$required" ]; then
+    echo "0"
+  else
+    [ "$VERBOSE_MODE" -eq "1" ] && logger -s "Not enough memory available" \
+                                          -p daemon.err
+    echo "1"
+  fi
+}
 
 collect_data(){
   n=0
-  [ "$verbose_mode" = "1" ] && logger -s "Collecting NetJSON Monitoring data" \
-                                      -p daemon.info
+  [ "$VERBOSE_MODE" -eq "1" ] && logger -s "Collecting NetJSON Monitoring data" \
+                                        -p daemon.info
   until [ "$n" -ge 5 ]
   do
     echo "$(/usr/sbin/netjson_monitoring "$MONITORED_INTERFACES")" && break
     
     if [ "$n" -eq 5 ]; then
-      [ "$verbose_mode" = "1" ] && logger -s "Collecting data failed!" \
-                                          -p daemon.err
+      [ "$VERBOSE_MODE" -eq "1" ] && logger -s "Collecting data failed!" \
+                                            -p daemon.err
     fi
     n=$((n+1))
     sleep 5
   done
 }
 
-while true
-do
-  data="$(collect_data)"
-  #send data
-  response_code=$($CURL_COMMAND -H "Content-Type: application/json" \
-    -d "$data" \
-    -v "$URL")
-  if [ "$response_code" = "200" ]; then
-    [ "$verbose_mode" = "1" ] && logger -s "Data sent successfully." \
-                                        -p daemon.info
-  else
-    logger -s "Data not sent successfully. Response code is $response_code" \
-           -t openwisp_monitoring \
-           -p daemon.err
+set_url_and_curl(){
+  if [ -z "$BASE_URL" ]; then
+    echoerr "missing required --url option"
+    exit 1
+  fi
 
-    [ "$verbose_mode" = "0" ] && logger -s "Run with verbose mode to find more." \
-                                        -t openwisp_monitoring \
-                                        -p daemon.err
+  if [ -z "$UUID" ]; then
+    echoerr "missing required --uuid option"
+    exit 1
   fi
-  sleep "$INTERVAL" & wait $!
-done
+
+  if [ -z "$KEY" ]; then
+    echoerr "missing required --key option"
+    exit 1
+  fi
+
+  URL="$BASE_URL/api/v1/monitoring/device/$UUID/?key=$KEY"
+
+  CURL_COMMAND="curl -s -w "%{http_code}""
+  [ "$VERIFY_SSL" -eq "0" ] && CURL_COMMAND="$CURL_COMMAND -k"
+  [ "$VERBOSE_MODE" -eq "1" ] && CURL_COMMAND="$CURL_COMMAND -v"
+  MAX_RETRIES=${MAX_RETRIES:-5}
+  FAILING=0
+  return 0
+}
+
+save_data() {
+  while true
+  do
+    memory_available="$(check_available_memory)"
+    if [ "$memory_available" -eq "0" ]; then
+      data="$(collect_data)"
+      file_name="$(date -u +'%d-%m-%Y_%H:%M:%S')"
+      # make directory 
+      mkdir -p "$TMP_DIR"
+      # save data with file_name 
+      echo "$data" > "$TMP_DIR/$file_name"
+      [ "$VERBOSE_MODE" -eq "1" ] && logger -s "Data saved temporarily" \
+                                            -p daemon.info
+    fi
+    # get process id of the process sending data 
+    pid=$(pgrep -f "openwisp_monitoring.*--mode send")
+    kill -SIGUSR1 "$pid"
+    sleep "$INTERVAL"
+  done
+}
+
+handle_sigusr1() {
+  [ "$VERBOSE_MODE" -eq "1" ] && logger -s "SIGUSR1 received! Sending data" \
+                                        -p daemon.info
+  return 0
+}
+
+send_data() {
+  while true
+  do
+    for file in "$TMP_DIR"/*
+    do
+      if [ ! -f "$file" ]; then
+        [ "$VERBOSE_MODE" -eq "1" ] && logger -s "No data file found to send." \
+                                              -p daemon.info
+        trap handle_sigusr1 USR1
+        # SIGUSR1 signal received, interrupt sleep and continue sending data
+        sleep "$INTERVAL" & wait $!
+        continue
+      fi
+      trap "" USR1
+      basefilename=${file##*/}
+      filename=${basefilename%.*}
+      # extra zeroes are added for nanoseconds precision
+      url="$URL&time=$filename.000000"
+      # retry sending data in case of failure
+      failures=0
+      timeout=1
+      # check if the data is latest or old one
+      [ "$(echo "$TMP_DIR"/* | awk '{print $2}')" ] || url="$url&current=true"
+      while true
+      do
+        if [ "$failures" -eq "$MAX_RETRIES" ]; then
+          if [ "$VERBOSE_MODE" -eq "1" ]; then
+            logger -s "Data not sent successfully. Response received is $response_code" \
+                   -p daemon.err
+          elif [ "$FAILING" -eq "0" ]; then
+            FAILING=1
+            logger -s "Data not sent successfully. Response received is $response_code" \
+                      "Run with verbose mode to find more." \
+                   -t openwisp_monitoring \
+                   -p daemon.err
+          fi
+          break
+        fi
+        # send data
+        data=$(cat "$file")
+        response_code=$($CURL_COMMAND -H "Content-Type: application/json" -d "$data" "$url")
+        if [ "$response_code" = "200" ]; then
+          if [ "$VERBOSE_MODE" -eq "1" ]; then
+            logger -s "Data sent successfully." \
+                   -p daemon.info
+          elif [ "$FAILING" -eq "1" ]; then
+            logger -s "Data sent successfully" \
+                   -t openwisp_monitoring \
+                   -p daemon.info
+            FAILING=0
+          fi
+          # remove saved data
+          rm "$file"
+          break
+        else
+          timeout=$((timeout*2))
+          [ "$VERBOSE_MODE" -eq "1" ] && logger -s "Data not sent successfully. Retrying in $timeout seconds" \
+                                                -p daemon.warn
+          failures=$((failures+1))
+          sleep "$timeout"
+        fi
+      done
+      # retry sending same data again in next cycle
+      [ "$failures" -eq "$MAX_RETRIES" ] && break
+    done
+  done
+}
+
+if [ -z "$MODE" ]; then
+  echoerr "missing required --mode option"
+  exit 1
+fi
+
+if [ "$MODE" = "collect" ]; then
+  MONITORED_INTERFACES=${MONITORED_INTERFACES:-*}
+  # remove double quotes from interfaces 
+  MONITORED_INTERFACES=$(echo "$MONITORED_INTERFACES" | tr -d '"')
+  save_data
+elif [ "$MODE" = "send" ]; then
+  VERIFY_SSL=${VERIFY_SSL:-0}
+  set_url_and_curl && send_data
+else
+  echoerr "The supplied mode is invalid. Only send and collect are allowed"
+  exit 1
+fi
diff --git a/openwrt-openwisp-monitoring/files/monitoring.config b/openwrt-openwisp-monitoring/files/monitoring.config
@@ -2,3 +2,5 @@ config monitoring 'monitoring'
 	option monitored_interfaces '*'
 	option interval '300'
 	option verbose_mode '0'
+	option required_memory '0.05'
+	option max_retries '5'
diff --git a/openwrt-openwisp-monitoring/files/monitoring.init b/openwrt-openwisp-monitoring/files/monitoring.init
@@ -55,6 +55,8 @@ start_service() {
   config_get monitored_interfaces monitoring monitored_interfaces "*"
   config_get interval monitoring interval "300"
   config_get_bool verbose_mode monitoring verbose_mode "0"
+  config_get required_memory monitoring required_memory "0.05"
+  config_get max_retries monitoring max_retries "5"
 
   interval="$(time_to_seconds "$interval")"
   if [ "$interval" -lt 1 ]; then
@@ -66,13 +68,21 @@ start_service() {
   interval="--interval $interval"
   monitored_interfaces="--monitored_interfaces \"$monitored_interfaces\""
   verbose="--verbose_mode $verbose_mode"
+  required_memory="--required_memory $required_memory"
+  max_retries="--max_retries $max_retries"
 
-  procd_open_instance "openwisp_monitoring_monitoring"
-  procd_set_param command $PROG $base_url $uuid $key $verify_ssl $interval $monitored_interfaces $verbose
+  procd_open_instance "openwisp_monitoring_collect_data"
+  procd_set_param command $PROG $interval $monitored_interfaces $verbose $required_memory --mode collect
   procd_set_param respawn "${respawn_threshold:-3600}" "${respawn_timeout:-5}" "${respawn_retry:-5}"
-  [ "$verbose_mode" = "1" ] && procd_set_param stdout 1
-  [ "$verbose_mode" = "1" ] && procd_set_param stderr 1
+  [ "$verbose_mode" -eq "1" ] && procd_set_param stdout 1 && procd_set_param stderr 1
   procd_close_instance
+
+  procd_open_instance "openwisp_monitoring_send_data"
+  procd_set_param command $PROG $base_url $uuid $key $verify_ssl $interval $verbose $max_retries --mode send
+  procd_set_param respawn "${respawn_threshold:-3600}" "${respawn_timeout:-5}" "${respawn_retry:-5}"
+  [ "$verbose_mode" -eq "1" ] && procd_set_param stdout 1 && procd_set_param stderr 1
+  procd_close_instance
+
   logger -s "$PROG_NAME started" \
          -t openwisp_monitoring \
          -p daemon.info
diff --git a/runbuild b/runbuild
@@ -11,6 +11,7 @@ BUILD_DIR=${BUILD_DIR-./build}
 DOWNLOADS_DIR=${DOWNLOADS_DIR-./downloads}
 START_TIME=${START_TIME-$(date +"%Y-%m-%d-%H%M%S")}
 VERSIONED_DIR="$DOWNLOADS_DIR/$START_TIME"
+COMPILE_TARGET=${COMPILE_TARGET-mips_24kc}
 LATEST_LINK="$DOWNLOADS_DIR/latest"
 CORES=${CORES:-1}
 CURRENT_DIR=$(pwd)
@@ -52,7 +53,7 @@ fi
 
 make -j$CORES package/openwrt-openwisp-monitoring/compile || exit 1
 
-mv $BUILD_DIR/openwrt/bin/packages/mips_24kc/openwisp $VERSIONED_DIR
+mv $BUILD_DIR/openwrt/bin/packages/$COMPILE_TARGET/openwisp $VERSIONED_DIR
 
 rm $LATEST_LINK || true
 ln -s $VERSIONED_DIR $LATEST_LINK
