feat: added restart alerting in development

terraform/main.tf

@@ -181,6 +181,26 @@ module "db-sns" {
   alert_pagerduty_integration_url = var.alert_pagerduty_integration_url
 }
 
+module "backoffice-restart-alerts" {
+  source           = "./modules/ecs-restart-alerts"
+  count            = var.enable_restart_alerts ? 1 : 0
+  service_name     = "backoffice"
+  cluster_arn      = module.droits-ecs-cluster.ecs_cluster_arn
+  alerts_topic_arn = module.backoffice-sns.alerts-topic-arn
+
+  depends_on = [module.droits-ecs-cluster, module.backoffice-sns]
+}
+
+module "webapp-restart-alerts" {
+  source           = "./modules/ecs-restart-alerts"
+  count            = var.enable_restart_alerts ? 1 : 0
+  service_name     = "webapp"
+  cluster_arn      = module.droits-ecs-cluster.ecs_cluster_arn
+  alerts_topic_arn = module.webapp-sns.alerts-topic-arn
+
+  depends_on = [module.droits-ecs-cluster, module.webapp-sns]
+}
+
 module "elasticache" {
   source           = "./modules/elasticache"
   vpc_id           = module.vpc.vpc_id

terraform/modules/ecs-cluster/outputs.tf

@@ -1,3 +1,7 @@
 output "ecs_cluster_id" {
   value = aws_ecs_cluster.droits-ecs-cluster.id
-}
+}
+
+output "ecs_cluster_arn" {
+  value = aws_ecs_cluster.droits-ecs-cluster.arn
+}

terraform/modules/ecs-restart-alerts/main.tf

@@ -0,0 +1,90 @@
+locals {
+  alarm_name = "ecs-${terraform.workspace}-droits-${var.service_name}-container-restarted"
+}
+
+resource "aws_cloudwatch_event_rule" "task_stopped" {
+  name        = "ecs-${terraform.workspace}-droits-${var.service_name}-task-stopped"
+  description = "ECS container stopped"
+
+  event_pattern = jsonencode({
+    source        = ["aws.ecs"]
+    "detail-type" = ["ECS Task State Change"]
+    detail = {
+      clusterArn = [var.cluster_arn]
+      group      = ["service:${var.service_name}"]
+      lastStatus = ["STOPPED"]
+    }
+  })
+}
+
+resource "aws_cloudwatch_event_target" "task_stopped" {
+  rule      = aws_cloudwatch_event_rule.task_stopped.name
+  target_id = "${var.service_name}-task-stopped-alerts"
+  arn       = var.alerts_topic_arn
+
+  input_transformer {
+    input_paths = {
+      time     = "$.time"
+      account  = "$.account"
+      region   = "$.region"
+      taskArn  = "$.detail.taskArn"
+      stopCode = "$.detail.stopCode"
+      reason   = "$.detail.stoppedReason"
+    }
+
+    input_template = <<-EOT
+      {
+        "AlarmName": "${local.alarm_name}",
+        "AlarmDescription": "Container has restarted",
+        "NewStateValue": "ALARM",
+        "NewStateReason": "Task <taskArn> stopped with code <stopCode>: <reason>",
+        "StateChangeTime": "<time>",
+        "Region": "<region>",
+        "AWSAccountId": "<account>"
+      }
+    EOT
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "task_running" {
+  name        = "ecs-${terraform.workspace}-droits-${var.service_name}-task-running"
+  description = "ECS container running"
+
+  event_pattern = jsonencode({
+    source        = ["aws.ecs"]
+    "detail-type" = ["ECS Task State Change"]
+    detail = {
+      clusterArn    = [var.cluster_arn]
+      group         = ["service:${var.service_name}"]
+      lastStatus    = ["RUNNING"]
+      desiredStatus = ["RUNNING"]
+    }
+  })
+}
+
+resource "aws_cloudwatch_event_target" "task_running" {
+  rule      = aws_cloudwatch_event_rule.task_running.name
+  target_id = "${var.service_name}-task-running-alerts"
+  arn       = var.alerts_topic_arn
+
+  input_transformer {
+    input_paths = {
+      time    = "$.time"
+      account = "$.account"
+      region  = "$.region"
+      taskArn = "$.detail.taskArn"
+    }
+
+    input_template = <<-EOT
+      {
+        "AlarmName": "${local.alarm_name}",
+        "AlarmDescription": "Container has restarted",
+        "NewStateValue": "OK",
+        "NewStateReason": "Task <taskArn> is running.",
+        "StateChangeTime": "<time>",
+        "Region": "<region>",
+        "AWSAccountId": "<account>"
+      }
+    EOT
+  }
+}

terraform/modules/ecs-restart-alerts/variables.tf

@@ -0,0 +1,14 @@
+variable "service_name" {
+  type        = string
+  description = "The name of the ECS service to watch for container restarts. E.g webapp"
+}
+
+variable "cluster_arn" {
+  type        = string
+  description = "The ARN of the ECS cluster the service runs in"
+}
+
+variable "alerts_topic_arn" {
+  type        = string
+  description = "The ARN of the SNS topic that forwards alerts to PagerDuty"
+}

terraform/modules/sns/alerts-topic.tf

@@ -67,5 +67,23 @@ data "aws_iam_policy_document" "alerts" {
     sid = "__default_statement_ID"
   }
 
+  statement {
+    sid    = "PublishEventsToTopic"
+    effect = "Allow"
+
+    actions = [
+      "SNS:Publish",
+    ]
+
+    principals {
+      type        = "Service"
+      identifiers = ["events.amazonaws.com"]
+    }
+
+    resources = [
+      aws_sns_topic.alerts.arn
+    ]
+  }
+
   depends_on = [aws_sns_topic.alerts]
 }

terraform/tfvars/dev.tfvars

@@ -33,6 +33,7 @@ ssl_domains = [
 lb_ssl_policy = "ELBSecurityPolicy-FS-1-2-2019-08"
 
 enable_alerts                                      = false
+enable_restart_alerts                              = true
 percentage_cpu_utilization_high_threshold          = 90
 percentage_memory_utilization_high_threshold       = 90
 cpu_utilisation_duration_in_seconds_to_evaluate    = 300

terraform/variables.tf

@@ -65,6 +65,12 @@ variable "enable_alerts" {
   default     = false
 }
 
+variable "enable_restart_alerts" {
+  type        = bool
+  description = "When enabled ECS container restart events are sent to the Alerts SNS Topic"
+  default     = false
+}
+
 variable "root_domain_name" {
   type        = string
   description = "The root domain name for DROITS"