diff --git a/README.md b/README.md index 3ef84841..5e7f31cc 100755 --- a/README.md +++ b/README.md @@ -23,6 +23,15 @@ Dow Jones Hammer documentation is available via GitHub Pages at [https://dowjone * [S3 Unencrypted Buckets](https://dowjones.github.io/hammer/playbook11_s3_unencryption.html) * [RDS Unencrypted Instances](https://dowjones.github.io/hammer/playbook12_rds_unencryption.html) * [AMIs Public Access](https://dowjones.github.io/hammer/playbook13_amis_public_access.html) +* [Redshift Unencrypted Clusters](https://dowjones.github.io/hammer/playbook15_redshift_unencryption.html) +* [Redshift Public Clusters](https://dowjones.github.io/hammer/playbook16_redshift_public_clusters.html) +* [Redshift Logging Issues](https://dowjones.github.io/hammer/playbook17_redshift_audit_logging.html) +* [ECS Logging Issues](https://dowjones.github.io/hammer/playbook18_ecs_logging.html) +* [ECS Privileged Access Issues](https://dowjones.github.io/hammer/playbook19_ecs_privileged_access.html) +* [ECS Exteranal Image Source Issues](https://dowjones.github.io/hammer/playbook20_ecs_external_image_source.html) +* [Elasticsearch Unencrypted Domains](https://dowjones.github.io/hammer/playbook21_elasticsearch_unencryption.html) +* [Elasticsearch Public Domains](https://dowjones.github.io/hammer/playbook22_elasticsearch_public_access.html) +* [Elasticsearch Logging Issues](https://dowjones.github.io/hammer/playbook23_elasticsearch_logging.html) ## Technologies * Python 3.6 diff --git a/deployment/build_packages.sh b/deployment/build_packages.sh index 2e00c69c..d0c53c5a 100755 --- a/deployment/build_packages.sh +++ b/deployment/build_packages.sh @@ -23,7 +23,7 @@ SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )" PACKAGES_DIR="${SCRIPT_PATH}/packages/" LIBRARY="${SCRIPT_PATH}/../hammer/library" -LAMBDAS="ami-info logs-forwarder ddb-tables-backup sg-issues-identification s3-acl-issues-identification s3-policy-issues-identification iam-keyrotation-issues-identification iam-user-inactive-keys-identification cloudtrails-issues-identification ebs-unencrypted-volume-identification ebs-public-snapshots-identification rds-public-snapshots-identification sqs-public-policy-identification s3-unencrypted-bucket-issues-identification rds-unencrypted-instance-identification ami-public-access-issues-identification api" +LAMBDAS="ami-info logs-forwarder ddb-tables-backup sg-issues-identification s3-acl-issues-identification s3-policy-issues-identification iam-keyrotation-issues-identification iam-user-inactive-keys-identification cloudtrails-issues-identification ebs-unencrypted-volume-identification ebs-public-snapshots-identification rds-public-snapshots-identification sqs-public-policy-identification s3-unencrypted-bucket-issues-identification rds-unencrypted-instance-identification ami-public-access-issues-identification api ecs-privileged-access-issues-identification ecs-logging-issues-identification ecs-external-image-source-issues-identification redshift-audit-logging-issues-identification redshift-unencrypted-cluster-identification redshift-cluster-public-access-identification elasticsearch-domain-logging-issues-identification elasticsearch-unencrypted-domain-identification elasticsearch-public-access-domain-identification" pushd "${SCRIPT_PATH}" > /dev/null pushd ../hammer/identification/lambdas > /dev/null diff --git a/deployment/cf-templates/ddb.json b/deployment/cf-templates/ddb.json index 9b55f4d5..ba3e13c6 100755 --- a/deployment/cf-templates/ddb.json +++ b/deployment/cf-templates/ddb.json @@ -24,7 +24,7 @@ } ], "ProvisionedThroughput": { - "ReadCapacityUnits": "25", + "ReadCapacityUnits": "50", "WriteCapacityUnits": "2" }, "SSESpecification": { @@ -480,6 +480,291 @@ }, "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "api-requests" ] ]} } + }, + "DynamoDBECSPrivilegedAccess": { + "Type": "AWS::DynamoDB::Table", + "DeletionPolicy": "Retain", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "ecs-privileged-access" ] ]} + } + }, + "DynamoDBECSLogging": { + "Type": "AWS::DynamoDB::Table", + "DeletionPolicy": "Retain", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "ecs-logging" ] ]} + } + }, + "DynamoDBECSExternalImageSource": { + "Type": "AWS::DynamoDB::Table", + "DeletionPolicy": "Retain", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "ecs-external-image-source" ] ]} + } + }, + "DynamoDBRedshiftLogging": { + "Type": "AWS::DynamoDB::Table", + "DeletionPolicy": "Retain", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "redshift-logging" ] ]} + } + }, + "DynamoDBRedshiftClusterEncryption": { + "Type": "AWS::DynamoDB::Table", + "DeletionPolicy": "Retain", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "redshift-unencrypted" ] ]} + } + }, + "DynamoDBRedshiftClusterPublicAccess": { + "Type": "AWS::DynamoDB::Table", + "DeletionPolicy": "Retain", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "redshift-public-access" ] ]} + } + }, + "DynamoDBESLoggingRequests": { + "Type": "AWS::DynamoDB::Table", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "es-domain-logging" ] ]} + } + }, + "DynamoDBESEncryptionRequests": { + "Type": "AWS::DynamoDB::Table", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "es-unencrypted-domain" ] ]} + } + }, + "DynamoDBESPublicAccessRequests": { + "Type": "AWS::DynamoDB::Table", + "DependsOn": ["DynamoDBCredentials"], + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "account_id", + "AttributeType": "S" + }, + { + "AttributeName": "issue_id", + "AttributeType": "S" + } + ], + "KeySchema": [ + { + "AttributeName": "account_id", + "KeyType": "HASH" + }, + { + "AttributeName": "issue_id", + "KeyType": "RANGE" + } + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": "10", + "WriteCapacityUnits": "2" + }, + "TableName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "es-public-access-domain" ] ]} + } } } } diff --git a/deployment/cf-templates/identification-crossaccount-role.json b/deployment/cf-templates/identification-crossaccount-role.json index e5c16c32..bb1ae517 100755 --- a/deployment/cf-templates/identification-crossaccount-role.json +++ b/deployment/cf-templates/identification-crossaccount-role.json @@ -47,7 +47,7 @@ "ec2:DescribeInstances", "ec2:DescribeRouteTables", "ec2:DescribeSubnets", - "ec2:DescribeImages", + "ec2:DescribeImages" ], "Resource": "*" }, @@ -63,6 +63,18 @@ ], "Resource": "*" }, + { + "Sid": "RedshiftIssues", + "Effect": "Allow", + "Action": [ + "redshift:DescribeClusterSecurityGroups", + "redshift:DescribeClusterParameterGroups", + "redshift:DescribeLoggingStatus", + "redshift:GetClusterCredentials", + "redshift:DescribeClusters" + ], + "Resource": "*" + }, { "Sid": "IamIssues", "Effect": "Allow", @@ -115,6 +127,39 @@ "sqs:ListQueueTags" ], "Resource": "*" + }, + { + "Sid": "ECSIssues", + "Effect": "Allow", + "Action": [ + "ecs:Describe*", + "ecs:List*" + ], + "Resource": "*" + }, + { + "Sid": "RedshiftIssues", + "Effect": "Allow", + "Action": [ + "redshift:DescribeClusterSecurityGroups", + "redshift:DescribeClusterParameterGroups", + "redshift:DescribeLoggingStatus", + "redshift:GetClusterCredentials", + "redshift:DescribeClusters" + ], + "Resource": "*" + }, + { + "Sid": "ESIssues", + "Effect": "Allow", + "Action": [ + "es:ListDomainNames", + "es:DescribeElasticsearchDomain", + "es:DescribeElasticsearchDomainConfig", + "es:DescribeElasticsearchDomains", + "es:ListTags" + ], + "Resource": "*" } ] } diff --git a/deployment/cf-templates/identification-role.json b/deployment/cf-templates/identification-role.json index 288897f2..b2ccfb0f 100755 --- a/deployment/cf-templates/identification-role.json +++ b/deployment/cf-templates/identification-role.json @@ -175,6 +175,39 @@ ], "Resource": "*" }, + { + "Sid": "ECSIssues", + "Effect": "Allow", + "Action": [ + "ecs:Describe*", + "ecs:List*" + ], + "Resource": "*" + }, + { + "Sid": "RedshiftIssues", + "Effect": "Allow", + "Action": [ + "redshift:DescribeClusterSecurityGroups", + "redshift:DescribeClusterParameterGroups", + "redshift:DescribeLoggingStatus", + "redshift:GetClusterCredentials", + "redshift:DescribeClusters" + ], + "Resource": "*" + }, + { + "Sid": "ESIssues", + "Effect": "Allow", + "Action": [ + "es:ListDomainNames", + "es:DescribeElasticsearchDomain", + "es:DescribeElasticsearchDomainConfig", + "es:DescribeElasticsearchDomains", + "es:ListTags" + ], + "Resource": "*" + }, { "Sid": "IAMassumeCrossAccountRole", "Effect": "Allow", diff --git a/deployment/cf-templates/identification.json b/deployment/cf-templates/identification.json index d83355db..eaddc546 100755 --- a/deployment/cf-templates/identification.json +++ b/deployment/cf-templates/identification.json @@ -27,7 +27,16 @@ "SourceIdentificationEBSVolumes", "SourceIdentificationEBSSnapshots", "SourceIdentificationRDSSnapshots", - "SourceIdentificationAMIPublicAccess" + "SourceIdentificationAMIPublicAccess", + "SourceIdentificationRedshiftPublicAccess", + "SourceIdentificationRedshiftClusterEncryption", + "SourceIdentificationRedshiftLogging", + "SourceIdentificationECSPrivilegedAccess", + "SourceIdentificationECSLogging", + "SourceIdentificationECSExternalImageSource", + "SourceIdentificationElasticSearchLogging", + "SourceIdentificationElasticSearchEncryption", + "SourceIdentificationElasticSearchPublicAccess" ] }, { @@ -92,6 +101,33 @@ }, "SourceIdentificationAMIPublicAccess":{ "default": "Relative path to Public AMI sources" + }, + "SourceIdentificationRedshiftPublicAccess":{ + "default": "Relative path to publicly accessible Redshift Cluster sources" + }, + "SourceIdentificationRedshiftClusterEncryption":{ + "default": "Relative path to unencrypted Redshift Cluster sources" + }, + "SourceIdentificationRedshiftLogging": { + "default": "Relative path to disabled logging Redshift Cluster sources" + }, + "SourceIdentificationECSPrivilegedAccess":{ + "default": "Relative path to privileged access issue ECS sources" + }, + "SourceIdentificationECSLogging":{ + "default": "Relative path to disabled logging ECS sources" + }, + "SourceIdentificationECSExternalImageSource":{ + "default": "Relative path to external image issue ECS sources" + }, + "SourceIdentificationElasticSearchLogging":{ + "dafault": "Relative path to Elasticsearch domain logging sources" + }, + "SourceIdentificationElasticSearchEncryption":{ + "dafault": "Relative path to Unencrypted Elasticsearch domain sources" + }, + "SourceIdentificationElasticSearchPublicAccess":{ + "dafault": "Relative path to Unencrypted Elasticsearch domain public access sources" } } } @@ -188,7 +224,43 @@ "SourceIdentificationRDSEncryption": { "Type": "String", "Default": "rds-unencrypted-instance-identification.zip" - } + }, + "SourceIdentificationRedshiftPublicAccess": { + "Type": "String", + "Default": "redshift-cluster-public-access-identification.zip" + }, + "SourceIdentificationRedshiftClusterEncryption": { + "Type": "String", + "Default": "redshift-unencrypted-cluster-identification.zip" + }, + "SourceIdentificationRedshiftLogging": { + "Type": "String", + "Default": "redshift-audit-logging-issues-identification.zip" + }, + "SourceIdentificationECSPrivilegedAccess": { + "Type": "String", + "Default": "ecs-privileged-access-issues-identification.zip" + }, + "SourceIdentificationECSLogging": { + "Type": "String", + "Default": "ecs-logging-issues-identification.zip" + }, + "SourceIdentificationECSExternalImageSource": { + "Type": "String", + "Default": "ecs-image-source-issues-identification.zip" + }, + "SourceIdentificationElasticSearchLogging": { + "Type": "String", + "Default": "elasticsearch-domain-logging-issues-identification.zip" + }, + "SourceIdentificationElasticSearchEncryption": { + "Type": "String", + "Default": "elasticsearch-unencrypted-domain-identification.zip" + }, + "SourceIdentificationElasticSearchPublicAccess": { + "Type": "String", + "Default": "elasticsearch-public-access-domain-identification.zip" + } }, "Conditions": { "LambdaSubnetsEmpty": { @@ -200,9 +272,6 @@ }, "Mappings": { "NamingStandards": { - "IdentificationMetricsNamespace": { - "value": "HammerIdentification" - }, "SNSTopicNameIdentificationErrors": { "value": "identification-errors" }, @@ -245,6 +314,33 @@ "IdentificationMetricRDSEncryptionError": { "value": "RDSEncryptionError" }, + "IdentificationMetricRedshiftPublicAccessError": { + "value": "RedshiftPublicAccessError" + }, + "IdentificationMetricRedshiftClusterEncryptionError": { + "value": "RedshiftClusterEncryptionError" + }, + "IdentificationMetricRedshiftLoggingError": { + "value": "RedshiftLoggingError" + }, + "IdentificationMetricECSPrivilegedAccessError": { + "value": "ECSPrivilegedAccessError" + }, + "IdentificationMetricECSLoggingError": { + "value": "ECSLoggingError" + }, + "IdentificationMetricECSExternalImageSourceError": { + "value": "ECSExternalImageSourceError" + }, + "IdentificationMetricESLoggingError": { + "value": "ESLoggingError" + }, + "IdentificationMetricESEncryptionError": { + "value": "ESEncryptionError" + }, + "IdentificationMetricESPublicAccessError": { + "value": "ESPublicAccessError" + }, "SNSDisplayNameSecurityGroups": { "value": "describe-security-groups-sns" }, @@ -323,89 +419,131 @@ "SNSTopicNameRDSEncryption": { "value": "describe-rds-encryption-lambda" }, - "LogsForwarderLambdaFunctionName": { - "value": "logs-forwarder" + "SNSDisplayNameRedshiftPublicAccess": { + "value": "describe-redshift-public-access-sns" }, - "BackupDDBLambdaFunctionName": { - "value": "backup-ddb" + "SNSTopicNameRedshiftPublicAccess": { + "value": "describe-redshift-public-access-lambda" + }, + "SNSDisplayNameRedshiftClusterEncryption": { + "value": "describe-redshift-cluster-encryption-sns" + }, + "SNSTopicNameRedshiftClusterEncryption": { + "value": "describe-redshift-cluster-encryption-lambda" + }, + "SNSDisplayNameRedshiftLogging": { + "value": "describe-redshift-logging-sns" }, - "InitiateSecurityGroupLambdaFunctionName": { - "value": "initiate-security-groups" + "SNSTopicNameRedshiftLogging": { + "value": "describe-redshift-logging-lambda" }, - "IdentifySecurityGroupLambdaFunctionName": { - "value": "describe-security-groups" + "SNSDisplayNameECSPrivilegedAccess": { + "value": "describe-ecs-privileged-access-sns" }, - "InitiateCloudTrailsLambdaFunctionName": { - "value": "initiate-cloudtrails" + "SNSTopicNameECSPrivilegedAccess": { + "value": "describe-ecs-privileged-access-lambda" }, - "IdentifyCloudTrailsLambdaFunctionName": { - "value": "describe-cloudtrails" + "SNSDisplayNameECSLogging": { + "value": "describe-ecs-logging-sns" }, - "InitiateS3ACLLambdaFunctionName": { - "value": "initiate-s3-acl" + "SNSTopicNameECSLogging": { + "value": "describe-ecs-logging-lambda" }, - "IdentifyS3ACLLambdaFunctionName": { - "value": "describe-s3-acl" + "SNSDisplayNameECSExternalImageSource": { + "value": "describe-ecs-external-image-source-sns" }, - "InitiateS3PolicyLambdaFunctionName": { - "value": "initiate-s3-policy" + "SNSTopicNameECSExternalImageSource": { + "value": "describe-ecs-external-image-source-lambda" }, - "IdentifyS3PolicyLambdaFunctionName": { - "value": "describe-s3-policy" + "SNSDisplayNameESLogging": { + "value": "describe-es-logging-sns" }, - "InitiateIAMUserKeysRotationLambdaFunctionName": { - "value": "initiate-iam-user-keys-rotation" + "SNSTopicNameESLogging": { + "value": "describe-es-logging-lambda" }, - "IdentifyIAMUserKeysRotationLambdaFunctionName": { - "value": "describe-iam-user-keys-rotation" + "SNSDisplayNameESEncryption": { + "value": "describe-es-encryption-sns" + }, + "SNSTopicNameESEncryption": { + "value": "describe-es-encryption-lambda" + }, + "SNSDisplayNameESPublicAccess": { + "value": "describe-es-public-access-sns" + }, + "SNSTopicNameESPublicAccess": { + "value": "describe-es-public-access-lambda" + }, + "LogsForwarderLambdaFunctionName": { + "value": "logs-forwarder" + }, + "BackupDDBLambdaFunctionName": { + "value": "backup-ddb" }, - "InitiateIAMUserInactiveKeysLambdaFunctionName": { - "value": "initiate-iam-user-inactive-keys" + "SecurityGroupLambdaFunctionName": { + "value": "security-groups" }, - "IdentifyIAMUserInactiveKeysLambdaFunctionName": { - "value": "describe-iam-user-inactive-keys" + "CloudTrailsLambdaFunctionName": { + "value": "cloudtrails" }, - "InitiateEBSVolumesLambdaFunctionName": { - "value": "initiate-ebs-unencrypted-volumes" + "S3ACLLambdaFunctionName": { + "value": "s3-acl" }, - "IdentifyEBSVolumesLambdaFunctionName": { - "value": "describe-ebs-unencrypted-volumes" + "S3PolicyLambdaFunctionName": { + "value": "s3-policy" }, - "InitiateEBSSnapshotsLambdaFunctionName": { - "value": "initiate-ebs-public-snapshots" + "IAMUserKeysRotationLambdaFunctionName": { + "value": "iam-user-keys-rotation" }, - "IdentifyEBSSnapshotsLambdaFunctionName": { - "value": "describe-ebs-public-snapshots" + "IAMUserInactiveKeysLambdaFunctionName": { + "value": "iam-user-inactive-keys" }, - "InitiateRDSSnapshotsLambdaFunctionName": { - "value": "initiate-rds-public-snapshots" + "EBSVolumesLambdaFunctionName": { + "value": "ebs-unencrypted-volumes" }, - "IdentifyRDSSnapshotsLambdaFunctionName": { - "value": "describe-rds-public-snapshots" + "EBSSnapshotsLambdaFunctionName": { + "value": "ebs-public-snapshots" }, - "InitiateAMIPublicAccessLambdaFunctionName": { - "value": "initiate-ami-public-access" + "RDSSnapshotsLambdaFunctionName": { + "value": "rds-public-snapshots" }, - "IdentifyAMIPublicAccessLambdaFunctionName": { - "value": "describe-ami-public-access" + "AMIPublicAccessLambdaFunctionName": { + "value": "ami-public-access" }, - "InitiateSQSPublicPolicyLambdaFunctionName": { - "value": "initiate-sqs-public-policy" + "SQSPublicPolicyLambdaFunctionName": { + "value": "sqs-public-policy" }, - "IdentifySQSPublicPolicyLambdaFunctionName": { - "value": "describe-sqs-public-policy" + "S3EncryptionLambdaFunctionName": { + "value": "s3-encryption" }, - "InitiateS3EncryptionLambdaFunctionName": { - "value": "initiate-s3-encryption" + "RDSEncryptionLambdaFunctionName": { + "value": "rds-encryption" }, - "IdentifyS3EncryptionLambdaFunctionName": { - "value": "describe-s3-encryption" + "RedshiftPublicAccessLambdaFunctionName": { + "value": "redshift-public-access" }, - "InitiateRDSEncryptionLambdaFunctionName": { - "value": "initiate-rds-encryption" + "RedshiftClusterEncryptionLambdaFunctionName": { + "value": "redshift-cluster-encryption" }, - "IdentifyRDSEncryptionLambdaFunctionName": { - "value": "describe-rds-encryption" + "RedshiftLoggingLambdaFunctionName": { + "value": "redshift-logging" + }, + "ECSPrivilegedAccessLambdaFunctionName": { + "value": "ecs-privileged-access" + }, + "ECSLoggingLambdaFunctionName": { + "value": "ecs-logging" + }, + "ECSExternalImageSourceLambdaFunctionName": { + "value": "ecs-external-image-source" + }, + "ESLoggingLambdaFunctionName": { + "value": "elasticsearch-logging" + }, + "ESEncryptionLambdaFunctionName": { + "value": "elasticsearch-encryption" + }, + "ESPublicAccessLambdaFunctionName": { + "value": "elasticsearch-public-access" } } }, @@ -591,17 +729,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "35 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "10 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": {"Ref": "SourceIdentificationSG"}, "InitiateLambdaDescription": "Lambda function for initiate to identify bad security groups", "EvaluateLambdaDescription": "Lambda function to describe security groups unrestricted access.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateSecurityGroupLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "SecurityGroupLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifySecurityGroupLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "SecurityGroupLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_sec_grps.lambda_handler", "EvaluateLambdaHandler": "describe_sec_grps_unrestricted_access.lambda_handler", @@ -631,17 +769,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "15 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "10 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationCloudTrails" }, "InitiateLambdaDescription": "Lambda function for initiate identification of CloudTrail issues", "EvaluateLambdaDescription": "Lambda function for initiate identification of CloudTrail issues", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateCloudTrailsLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "CloudTrailsLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyCloudTrailsLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "CloudTrailsLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_cloudtrails.lambda_handler", "EvaluateLambdaHandler": "describe_cloudtrails.lambda_handler", @@ -677,11 +815,11 @@ "IdentificationLambdaSource": { "Ref": "SourceIdentificationS3ACL" }, "InitiateLambdaDescription": "Lambda function for initiate to identify public s3 buckets.", "EvaluateLambdaDescription": "Lambda function to describe public s3 buckets.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateS3ACLLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "S3ACLLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyS3ACLLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "S3ACLLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_s3_bucket_acl.lambda_handler", "EvaluateLambdaHandler": "describe_s3_bucket_acl.lambda_handler", @@ -717,11 +855,11 @@ "IdentificationLambdaSource": { "Ref": "SourceIdentificationS3Policy" }, "InitiateLambdaDescription": "Lambda function for initiate to identify public s3 buckets.", "EvaluateLambdaDescription": "Lambda function to describe public s3 buckets.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateS3PolicyLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "S3PolicyLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyS3PolicyLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "S3PolicyLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_s3_bucket_policy.lambda_handler", "EvaluateLambdaHandler": "describe_s3_bucket_policy.lambda_handler", @@ -757,11 +895,11 @@ "IdentificationLambdaSource": { "Ref": "SourceIdentificationIAMUserKeysRotation" }, "InitiateLambdaDescription": "Lambda function for initiate to identify IAM user keys which to be rotate.", "EvaluateLambdaDescription": "Lambda function to describe IAM user keys to be rotated.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateIAMUserKeysRotationLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "IAMUserKeysRotationLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyIAMUserKeysRotationLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "IAMUserKeysRotationLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_iam_users_key_rotation.lambda_handler", "EvaluateLambdaHandler": "describe_iam_key_rotation.lambda_handler", @@ -797,11 +935,11 @@ "IdentificationLambdaSource": { "Ref": "SourceIdentificationIAMUserInactiveKeys" }, "InitiateLambdaDescription": "Lambda function for initiate to identify IAM user keys which last used.", "EvaluateLambdaDescription": "Lambda function to describe IAM user keys last used.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateIAMUserInactiveKeysLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "IAMUserInactiveKeysLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyIAMUserInactiveKeysLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "IAMUserInactiveKeysLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_iam_access_keys.lambda_handler", "EvaluateLambdaHandler": "describe_iam_accesskey_details.lambda_handler", @@ -831,17 +969,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "20 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "10 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationEBSVolumes" }, "InitiateLambdaDescription": "Lambda function for initiate to identify unencrypted EBS volumes.", "EvaluateLambdaDescription": "Lambda function to describe unencrypted ebs volumes.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateEBSVolumesLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "EBSVolumesLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyEBSVolumesLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "EBSVolumesLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_ebs_unencrypted_volumes.lambda_handler", "EvaluateLambdaHandler": "describe_ebs_unencrypted_volumes.lambda_handler", @@ -871,17 +1009,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "25 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationEBSSnapshots" }, "InitiateLambdaDescription": "Lambda function for initiate to identify public EBS snapshots.", "EvaluateLambdaDescription": "Lambda function to describe public ebs snapshots.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateEBSSnapshotsLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "EBSSnapshotsLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyEBSSnapshotsLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "EBSSnapshotsLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_ebs_public_snapshots.lambda_handler", "EvaluateLambdaHandler": "describe_ebs_public_snapshots.lambda_handler", @@ -917,11 +1055,11 @@ "IdentificationLambdaSource": { "Ref": "SourceIdentificationRDSSnapshots" }, "InitiateLambdaDescription": "Lambda function for initiate to identify public RDS snapshots.", "EvaluateLambdaDescription": "Lambda function to describe public RDS snapshots.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateRDSSnapshotsLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "RDSSnapshotsLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyRDSSnapshotsLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "RDSSnapshotsLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_rds_public_snapshots.lambda_handler", "EvaluateLambdaHandler": "describe_rds_public_snapshots.lambda_handler", @@ -951,17 +1089,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "40 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationSQSPublicPolicy" }, "InitiateLambdaDescription": "Lambda function for initiate to identify public SQS queues.", "EvaluateLambdaDescription": "Lambda function to describe public SQS queues.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateSQSPublicPolicyLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "SQSPublicPolicyLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifySQSPublicPolicyLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "SQSPublicPolicyLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_sqs_public_policy.lambda_handler", "EvaluateLambdaHandler": "describe_sqs_public_policy.lambda_handler", @@ -991,17 +1129,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "10 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationS3Encryption" }, "InitiateLambdaDescription": "Lambda function for initiate to identify S3 unencrypted buckets.", "EvaluateLambdaDescription": "Lambda function to describe un-encrypted S3 buckets.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateS3EncryptionLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "S3EncryptionLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyS3EncryptionLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "S3EncryptionLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_s3_encryption.lambda_handler", "EvaluateLambdaHandler": "describe_s3_encryption.lambda_handler", @@ -1031,17 +1169,17 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "40 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationRDSEncryption" }, "InitiateLambdaDescription": "Lambda function for initiate to identify unencrypted RDS instances.", "EvaluateLambdaDescription": "Lambda function to describe un-encrypted RDS instances.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateRDSEncryptionLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "RDSEncryptionLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyRDSEncryptionLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "RDSEncryptionLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_rds_instance_encryption.lambda_handler", "EvaluateLambdaHandler": "describe_rds_instance_encryption.lambda_handler", @@ -1071,21 +1209,21 @@ { "Ref": "ResourcesPrefix" }, { "Ref": "IdentificationIAMRole" } ] ]}, - "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "45 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, "LambdaSubnets": {"Ref": "LambdaSubnets"}, "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, "IdentificationLambdaSource": { "Ref": "SourceIdentificationAMIPublicAccess" }, "InitiateLambdaDescription": "Lambda function for initiate to identify public AMI access issues.", "EvaluateLambdaDescription": "Lambda function to describe public AMI issues.", - "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "InitiateAMIPublicAccessLambdaFunctionName", "value"] } ] + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "AMIPublicAccessLambdaFunctionName", "value"] } ] ]}, - "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, - { "Fn::FindInMap": ["NamingStandards", "IdentifyAMIPublicAccessLambdaFunctionName", "value"] } ] + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "AMIPublicAccessLambdaFunctionName", "value"] } ] ]}, "InitiateLambdaHandler": "initiate_to_desc_public_ami_issues.lambda_handler", "EvaluateLambdaHandler": "describe_public_ami_issues.lambda_handler", - "EvaluateLambdaMemorySize": 256, + "EvaluateLambdaMemorySize": 512, "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, "EventRuleDescription": "Hammer ScheduledRule to initiate public AMI access evaluations", "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationAMIPublicAccess"] ] }, @@ -1098,9 +1236,370 @@ "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} } } + }, + "StackEvaluateRedshiftPublicAccess": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + {"Ref": "ResourcesPrefix"}, + {"Ref": "IdentificationIAMRole"} + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationRedshiftPublicAccess" }, + "InitiateLambdaDescription": "Lambda function to initiate to identify Redshift public access issues.", + "EvaluateLambdaDescription": "Lambda function to describe Redshift public access issues.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "RedshiftPublicAccessLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "RedshiftPublicAccessLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_redshift_cluster_public_access.lambda_handler", + "EvaluateLambdaHandler": "describe_redshift_cluster_public_access.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Redshift public access issues evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationRedshiftPublicAccess"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameRedshiftPublicAccess", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameRedshiftPublicAccess", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateRedshiftClusterEncryption": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "30 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationRedshiftClusterEncryption" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify Redshift cluster is encrypted or not.", + "EvaluateLambdaDescription": "Lambda function to describe Redshift cluster is encrypted or not.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "RedshiftClusterEncryptionLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "RedshiftClusterEncryptionLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_redshift_encryption.lambda_handler", + "EvaluateLambdaHandler": "describe_redshift_encryption.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Redshift Cluster evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationRedshiftClusterEncryption"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameRedshiftClusterEncryption", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameRedshiftClusterEncryption", "value"] } ] + + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateRedshiftLogging": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "50 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationRedshiftLogging" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify Redshift logging issues.", + "EvaluateLambdaDescription": "Lambda function to describe Redshift logging issues.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "RedshiftLoggingLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "RedshiftLoggingLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_redshift_logging_issues.lambda_handler", + "EvaluateLambdaHandler": "describe_redshift_logging_issues.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Redshift logging issues evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationRedshiftLogging"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameRedshiftLogging", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameRedshiftLogging", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateECSPrivilegedAccess": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "50 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationECSPrivilegedAccess" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify ECS privileged access issues.", + "EvaluateLambdaDescription": "Lambda function to describe ECS privileged access issues.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "ECSPrivilegedAccessLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "ECSPrivilegedAccessLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_ecs_privileged_access_issues.lambda_handler", + "EvaluateLambdaHandler": "describe_ecs_privileged_access_issues.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate ECS privileged access evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationECSPrivilegedAccess"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameECSPrivilegedAccess", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameECSPrivilegedAccess", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateECSLogging": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": {"Ref": "SourceS3Bucket"}, + "IdentificationIAMRole": {"Fn::Join": ["", ["arn:aws:iam::", + {"Ref": "AWS::AccountId"}, + ":role/", + {"Ref": "ResourcesPrefix"}, + {"Ref": "IdentificationIAMRole"} + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["",["cron(","50 ",{"Ref": "IdentificationCheckRateExpression"},")"]]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": {"Ref": "SourceIdentificationECSLogging"}, + "InitiateLambdaDescription": "Lambda function for initiate to identify ECS logging enabled or not.", + "EvaluateLambdaDescription": "Lambda function to describe ECS logging enabled or not.", + "InitiateLambdaName": {"Fn::Join": ["",[{"Ref": "ResourcesPrefix"}, "initiate-", + { "Fn::FindInMap": ["NamingStandards","ECSLoggingLambdaFunctionName","value"]}] + ]}, + "EvaluateLambdaName": {"Fn::Join": ["",[{"Ref": "ResourcesPrefix"}, "describe-", + {"Fn::FindInMap": ["NamingStandards","ECSLoggingLambdaFunctionName","value"]}] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_ecs_logging_issues.lambda_handler", + "EvaluateLambdaHandler": "describe_ecs_logging_issues.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": {"Fn::GetAtt": ["LambdaLogsForwarder","Arn"]}, + "EventRuleDescription": "Hammer ScheduledRule to initiate ECS logging enabled or not evaluations", + "EventRuleName": {"Fn::Join": ["", [ {"Ref": "ResourcesPrefix"}, "InitiateEvaluationECSLogging"]]}, + "SNSDisplayName": {"Fn::Join": ["",[{"Ref": "ResourcesPrefix"}, + {"Fn::FindInMap": ["NamingStandards","SNSDisplayNameECSLogging","value"]}] + ]}, + "SNSTopicName": {"Fn::Join": ["",[{"Ref": "ResourcesPrefix"}, + {"Fn::FindInMap": ["NamingStandards","SNSTopicNameECSLogging","value"]}] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateECSExternalImageSource": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "50 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationECSExternalImageSource" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify ECS image source is internal or external.", + "EvaluateLambdaDescription": "Lambda function to describe ECS image source is internal or external.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "ECSExternalImageSourceLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "ECSExternalImageSourceLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_ecs_external_image_source_issues.lambda_handler", + "EvaluateLambdaHandler": "describe_ecs_external_image_source_issues.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate ECS image source evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationECSExternalImageSource"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameECSExternalImageSource", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameECSExternalImageSource", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateESEncryption": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "50 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationElasticSearchEncryption" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify unencrypted Elasticsearch domains.", + "EvaluateLambdaDescription": "Lambda function to describe un-encrypted Elasticsearch domains.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "ESEncryptionLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "ESEncryptionLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_elasticsearch_unencrypted_domains.lambda_handler", + "EvaluateLambdaHandler": "describe_elasticsearch_unencrypted_domains.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Elasticsearch domain encryption evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationESEncryption"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameESEncryption", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameESEncryption", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateESLogging": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "50 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationElasticSearchLogging" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify Elasticsearch domain logging issues.", + "EvaluateLambdaDescription": "Lambda function to describe Elasticsearch domain logging issues.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "ESLoggingLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "ESLoggingLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_elasticsearch_domains_logging_issues.lambda_handler", + "EvaluateLambdaHandler": "describe_elasticsearch_domains_logging_issues.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Elasticsearch domain logging evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationESLogging"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameESLogging", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameESLogging", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } + }, + "StackEvaluateESPublicAccess": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": {"Ref": "NestedStackTemplate"}, + "Parameters": { + "SourceS3Bucket": { "Ref": "SourceS3Bucket" }, + "IdentificationIAMRole": {"Fn::Join" : ["", [ "arn:aws:iam::", + { "Ref": "AWS::AccountId" }, + ":role/", + { "Ref": "ResourcesPrefix" }, + { "Ref": "IdentificationIAMRole" } + ] ]}, + "IdentificationCheckRateExpression": {"Fn::Join": ["", [ "cron(", "50 ", { "Ref": "IdentificationCheckRateExpression" }, ")" ] ]}, + "LambdaSubnets": {"Ref": "LambdaSubnets"}, + "LambdaSecurityGroups": {"Ref": "LambdaSecurityGroups"}, + "IdentificationLambdaSource": { "Ref": "SourceIdentificationElasticSearchPublicAccess" }, + "InitiateLambdaDescription": "Lambda function for initiate to identify publicly accessible Elasticsearch domains.", + "EvaluateLambdaDescription": "Lambda function to describe publicly accessible Elasticsearch domains.", + "InitiateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "initiate-", + { "Fn::FindInMap": ["NamingStandards", "ESPublicAccessLambdaFunctionName", "value"] } ] + ]}, + "EvaluateLambdaName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, "describe-", + { "Fn::FindInMap": ["NamingStandards", "ESPublicAccessLambdaFunctionName", "value"] } ] + ]}, + "InitiateLambdaHandler": "initiate_to_desc_elasticsearch_public_access_domains.lambda_handler", + "EvaluateLambdaHandler": "describe_elasticsearch_public_access_domains.lambda_handler", + "EvaluateLambdaMemorySize": 256, + "LambdaLogsForwarderArn": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }, + "EventRuleDescription": "Hammer ScheduledRule to initiate Elasticsearch domain public access evaluations", + "EventRuleName": {"Fn::Join" : ["", [{ "Ref": "ResourcesPrefix" }, "InitiateEvaluationESPublicAccess"] ] }, + "SNSDisplayName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSDisplayNameESPublicAccess", "value"] } ] + ]}, + "SNSTopicName": {"Fn::Join" : ["", [ { "Ref": "ResourcesPrefix" }, + { "Fn::FindInMap": ["NamingStandards", "SNSTopicNameESPublicAccess", "value"] } ] + ]}, + "SNSIdentificationErrors": {"Ref": "SNSIdentificationErrors"} + } + } } }, "Outputs": { "LambdaLogsForwarderArn": {"Value": { "Fn::GetAtt": ["LambdaLogsForwarder", "Arn"] }} } -} \ No newline at end of file +} diff --git a/deployment/cf-templates/reporting-remediation-crossaccount-role.json b/deployment/cf-templates/reporting-remediation-crossaccount-role.json index 4a1bda43..01ff4e02 100755 --- a/deployment/cf-templates/reporting-remediation-crossaccount-role.json +++ b/deployment/cf-templates/reporting-remediation-crossaccount-role.json @@ -138,6 +138,49 @@ "sqs:SetQueueAttributes" ], "Resource": "*" + }, + { + "Sid": "ECSIssues", + "Effect": "Allow", + "Action": [ + "ecs:Describe*", + "ecs:List*" + ], + "Resource": "*" + }, + { + "Sid": "RedshiftIssues", + "Effect": "Allow", + "Action": [ + "redshift:DescribeClusterSecurityGroups", + "redshift:DescribeClusterParameterGroups", + "redshift:DescribeLoggingStatus", + "redshift:GetClusterCredentials", + "redshift:DescribeClusters", + "redshift:EnableLogging", + "redshift:AuthorizeClusterSecurityGroupIngress", + "redshift:ModifyCluster", + "ec2:Describe*" + ], + "Resource": "*" + }, + { + "Sid": "ESIssues", + "Effect": "Allow", + "Action": [ + "es:ListDomainNames", + "es:DescribeElasticsearchDomain", + "es:DescribeElasticsearchDomainConfig", + "es:DescribeElasticsearchDomains", + "es:ListTags", + "es:UpdateElasticsearchDomainConfig", + "logs:DescribeLogGroups", + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents", + "logs:PutResourcePolicy" + ], + "Resource": "*" } ] } diff --git a/deployment/cf-templates/reporting-remediation-role.json b/deployment/cf-templates/reporting-remediation-role.json index e4840a55..258bc871 100755 --- a/deployment/cf-templates/reporting-remediation-role.json +++ b/deployment/cf-templates/reporting-remediation-role.json @@ -206,6 +206,49 @@ ], "Resource": "*" }, + { + "Sid": "ECSIssues", + "Effect": "Allow", + "Action": [ + "ecs:Describe*", + "ecs:List*" + ], + "Resource": "*" + }, + { + "Sid": "RedshiftIssues", + "Effect": "Allow", + "Action": [ + "redshift:DescribeClusterSecurityGroups", + "redshift:DescribeClusterParameterGroups", + "redshift:DescribeLoggingStatus", + "redshift:GetClusterCredentials", + "redshift:DescribeClusters", + "redshift:EnableLogging", + "redshift:AuthorizeClusterSecurityGroupIngress", + "redshift:ModifyCluster", + "ec2:Describe*" + ], + "Resource": "*" + }, + { + "Sid": "ESIssues", + "Effect": "Allow", + "Action": [ + "es:ListDomainNames", + "es:DescribeElasticsearchDomain", + "es:DescribeElasticsearchDomainConfig", + "es:DescribeElasticsearchDomains", + "es:ListTags", + "es:UpdateElasticsearchDomainConfig", + "logs:DescribeLogGroups", + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents", + "logs:PutResourcePolicy" + ], + "Resource": "*" + }, { "Sid": "IAMassumeCrossAccountRole", "Effect": "Allow", diff --git a/deployment/cf-templates/reporting-remediation.json b/deployment/cf-templates/reporting-remediation.json index 731b8b02..268096d6 100755 --- a/deployment/cf-templates/reporting-remediation.json +++ b/deployment/cf-templates/reporting-remediation.json @@ -170,7 +170,7 @@ "yum -y install openssl-devel\n", "# remove for following install to not fail\n", "yum -y remove ius-release\n", - "yum -y install https://centos7.iuscommunity.org/ius-release.rpm\n", + "yum -y install https://repo.ius.io/ius-release-el7.rpm --nogpgcheck\n", "yum -y install python36u python36u-pip python36u-devel\n", "pip3.6 install awscli\n", "rm -rf /hammer-correlation-engine\n", diff --git a/deployment/configs/config.json b/deployment/configs/config.json index 68bb3bef..2fcb37ae 100755 --- a/deployment/configs/config.json +++ b/deployment/configs/config.json @@ -53,7 +53,8 @@ "topic_name": "hammer-describe-s3-acl-lambda", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["publics3"] }, "secgrp_unrestricted_access": { "enabled": true, @@ -75,7 +76,8 @@ "reporting": false, "remediation": false, "remediation_accounts": ["210987654321", "654321210987"], - "remediation_retention_period": 21 + "remediation_retention_period": 21, + "labels": ["insecure-services"] }, "user_inactivekeys": { "enabled": true, @@ -85,7 +87,8 @@ "inactive_criteria_days": 1, "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["inactive-iam-keys"] }, "user_keysrotation": { "enabled": true, @@ -94,7 +97,8 @@ "rotation_criteria_days": 10, "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["iam-key-rotation"] }, "s3_bucket_policy": { "enabled": true, @@ -102,20 +106,23 @@ "topic_name": "hammer-describe-s3-policy-lambda", "reporting": false, "remediation": false, - "remediation_retention_period": 7 + "remediation_retention_period": 7, + "labels": ["publics3"] }, "cloudtrails": { "enabled": true, "ddb.table_name": "hammer-cloudtrails", "topic_name": "hammer-describe-cloudtrails-lambda", - "reporting": false + "reporting": false, + "labels": ["cloud-trail-disabled"] }, "ebs_unencrypted_volume": { "enabled": true, "ddb.table_name": "hammer-ebs-volumes-unencrypted", "topic_name": "hammer-describe-ebs-unencrypted-volumes-lambda", "accounts": ["123456789012", "210987654321"], - "reporting": false + "reporting": false, + "labels": ["unencrypted-ebs-volumes"] }, "ebs_public_snapshot": { "enabled": true, @@ -123,7 +130,8 @@ "topic_name": "hammer-describe-ebs-public-snapshots-lambda", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["public_snapshots"] }, "rds_public_snapshot": { "enabled": true, @@ -131,14 +139,17 @@ "topic_name": "hammer-describe-rds-public-snapshots-lambda", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["rds-public-snapshots"] }, "ec2_public_ami": { "enabled": true, "ddb.table_name": "hammer-ec2-public-ami", + "topic_name": "hammer-describe-ami-public-access-lambda", "reporting": false, "remediation": false, - "remediation_retention_period": 21 + "remediation_retention_period": 21, + "labels": ["public-ami"] }, "sqs_public_access": { "enabled": true, @@ -146,7 +157,8 @@ "topic_name": "hammer-describe-sqs-public-policy-lambda", "reporting": true, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["publicsqs"] }, "s3_encryption": { "enabled": true, @@ -154,12 +166,86 @@ "topic_name": "hammer-describe-s3-encryption-lambda", "reporting": true, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "labels": ["s3-unencrypted"] }, "rds_encryption": { "enabled": true, "ddb.table_name": "hammer-rds-unencrypted", "topic_name": "hammer-describe-rds-encryption-lambda", - "reporting": true + "reporting": true, + "labels": ["rds-unencrypted-instances"] + }, + "redshift_logging": { + "enabled": true, + "ddb.table_name": "hammer-redshift-logging", + "topic_name": "hammer-describe-redshift-logging-lambda", + "reporting": true, + "labels": ["redshift-logging"] + }, + "redshift_public_access": { + "enabled": true, + "ddb.table_name": "hammer-redshift-public-access", + "topic_name": "hammer-describe-redshift-public-access-lambda", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "labels": ["redshift-public-access"] + }, + "redshift_encryption": { + "enabled": true, + "ddb.table_name": "hammer-redshift-unencrypted", + "topic_name": "hammer-describe-redshift-cluster-encryption-lambda", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "labels": ["redshift-unencrypted-clusters"] + }, + "ecs_privileged_access": { + "enabled": true, + "ddb.table_name": "hammer-ecs-privileged-access", + "topic_name": "hammer-describe-ecs-privileged-access-lambda", + "reporting": true, + "labels": ["ecs-privileged-access"] + }, + "ecs_logging": { + "enabled": true, + "ddb.table_name": "hammer-ecs-logging", + "topic_name": "hammer-describe-ecs-logging-lambda", + "reporting": true, + "labels": ["ecs-logging"] + }, + "ecs_external_image_source": { + "enabled": true, + "topic_name": "hammer-describe-ecs-external-image-source-lambda", + "ddb.table_name": "hammer-ecs-external-image-source", + "reporting": true, + "safe_image_sources": ["amazonaws", "artifactory"], + "labels": ["ecs-external-image"] + }, + "es_domain_logging": { + "enabled": true, + "ddb.table_name": "hammer-es-domain-logging", + "topic_name": "hammer-describe-es-logging-lambda", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "labels": ["es-domain-logging"] + }, + "es_unencrypted_domain": { + "enabled": true, + "ddb.table_name": "hammer-es-unencrypted-domain", + "topic_name": "hammer-describe-es-encryption-lambda", + "reporting": true, + "labels": ["unencrypted-elasticsearch-domains"] + }, + "es_public_access_domain": { + "enabled": true, + "ddb.table_name": "hammer-es-public-access-domain", + "topic_name": "hammer-describe-es-public-access-lambda", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "labels": ["public-elasticsearch-domains"] } } diff --git a/deployment/configs/temp_whitelist_issues.json b/deployment/configs/temp_whitelist_issues.json new file mode 100644 index 00000000..678e80d2 --- /dev/null +++ b/deployment/configs/temp_whitelist_issues.json @@ -0,0 +1,91 @@ +{ + "__comment__": "Contains dictionary with security issues to quarantine (list of issues skipped now and will remediate in future) grouped by issue type and accounts. Put your account id as a key for desired security issue type and put a list with issues to ignore as a value.", + "cloudtrails": { + "__comment__": "Detects issues with CloudTrail (logging disabled or has issues with permissions). Key - account id, values - AWS regions.", + "123456789012": ["eu-west-1", "us-east-2"] + }, + "user_inactivekeys": { + "__comment__": "Detects IAM users with inactive access keys (not used more that definite number of days). Key - account id, values - IAM user names or access key ids.", + "123456789012": ["user1", "user2", "AKIAI6UV5TCF3NA223T1", "AKIAIG7Y36NN5DWX4NO3"] + }, + "user_keysrotation": { + "__comment__": "Detects IAM users expired access keys (created earlier than definite number of days). Key - account id, values - IAM user names or access key ids.", + "123456789012": ["user1", "user2", "AKIAI6UV5TCF3NA223T1", "AKIAIG7Y36NN5DWX4NO3"] + }, + "s3_bucket_acl": { + "__comment__": "Detects S3 buckets with public ACL (with AllUsers/AuthenticatedUsers groups in Grantee). Key - account id, values - S3 bucket names.", + "123456789012": ["public-site-bucket", "public-bucket-available-via-cloudfront"] + }, + "s3_bucket_policy": { + "__comment__": "Detects S3 buckets with public policy ('Allow' statements with '*' in Principal and not restricted by IP). Key - account id, values - S3 bucket names.", + "123456789012": ["public-site-bucket", "public-bucket-available-via-cloudfront"] + }, + "secgrp_unrestricted_access": { + "__comment__": "Detects security groups with world-wide open ports from the list. Key - account id, values - 1) security group ID or 2) VPC ID + security group Name separated by colon.", + "123456789012": ["sg-7c124307", "sg-2132a25b", "vpc-a372f3ca:default"] + }, + "ebs_unencrypted_volume": { + "__comment__": "Detects unencrypted EBS volumes. Key - account id, values - volume ids.", + "123456789012": ["vol-04ddaf8f2aef1b1f4", "vol-004156f485f6d57c7"] + }, + "ebs_public_snapshot": { + "__comment__": "Detects public EBS snapshots (with group 'all' in 'CreateVolumePermissions'). Key - account id, values - snapshot ids.", + "123456789012": ["snap-027927dbf368b3746", "snap-087534caad1ef1d0a"] + }, + "rds_public_snapshot":{ + "__comment__": "Detects public RDS snapshots (with 'all' in 'restore' attribute). Key - account id, values - snapshot ARNs.", + "123456789012": ["arn:aws:rds:eu-central-1:123456789012:snapshot:public", "arn:aws:rds:eu-west-1:123456789012:snapshot:rds:snapshot1"] + }, + "ec2_public_ami": { + "__comment__": "Detects public AMI issues (with 'all' in 'restore' attribute). Key - account id, values - AMI IDs.", + "123456789012": [""] + }, + "sqs_public_access":{ + "__comment__": "Detects public SQS polices (with 'all' in 'restore' attribute). Key - account id, values - SQS names.", + "123456789012": [""] + }, + "s3_encryption": { + "__comment__": "Detects Unencrypted s3 buckets (with 'all' in 'restore' attribute). Key - account id, values - S3 bucket names.", + "123456789012": [""] + }, + "rds_encryption": { + "__comment__": "Detects unencrypted RDS instances (with 'all' in 'restore' attribute). Key - account id, values - Instance ARNs.", + "123456789012": [""] + }, + "redshift_public_access":{ + "__comment__": "Detects publicly accessible Redshift Clusters.", + "123456789012": ["test-cluster"] + }, + "redshift_encryption":{ + "__comment__": "Detects unencrypted clusters.", + "123456789012": ["test-cluster"] + }, + "ecs_privileged_access":{ + "__comment__": "Detects ECS task definitions which are not enabled logging - task definitions ARNs.", + "1234567890123": ["arn:aws:ecs:us-east-1:1234567890123:task-definition/dev-admin:2993"] + }, + "ecs_logging":{ + "__comment__": "Detects ECS task definitions which are not enabled logging - task definitions ARNs.", + "1234567890123": ["arn:aws:ecs:us-east-1:1234567890123:task-definition/test-admin:2993"] + }, + "ecs_external_image_source":{ + "__comment__": "Detects ECS task definitions which are configured with external image source - task definitions ARNs.", + "1234567890123": ["arn:aws:ecs:us-east-1:1234567890123:task-definition/test-admin:2993"] + }, + "redshift_logging": { + "__comment__": "Detects Redshift clusters which are audit logging is not enabled.", + "123456789012": ["test-cluster"] + }, + "es_domain_logging": { + "__comment__": "Detects Elasticsearch domains which are not enabled logging - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, + "es_unencrypted_domain": { + "__comment__": "Detects Unencrypted Elasticsearch domains - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, + "es_public_access_domain": { + "__comment__": "Detects Unencrypted Elasticsearch publicly accessible domains - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + } +} diff --git a/deployment/configs/whitelist.json b/deployment/configs/whitelist.json index 3cd1ac81..02c4c7d4 100755 --- a/deployment/configs/whitelist.json +++ b/deployment/configs/whitelist.json @@ -36,7 +36,7 @@ "__comment__": "Detects public RDS snapshots (with 'all' in 'restore' attribute). Key - account id, values - snapshot ARNs.", "123456789012": ["arn:aws:rds:eu-central-1:123456789012:snapshot:public", "arn:aws:rds:eu-west-1:123456789012:snapshot:rds:snapshot1"] }, - "public_ami_issues": { + "ec2_public_ami": { }, "sqs_public_access":{ "__comment__": "Detects public SQS polices (with 'all' in 'restore' attribute). Key - account id, values - SQS ARNs.", @@ -45,5 +45,41 @@ "s3_encryption": { }, "rds_encryption": { - } + }, + "redshift_public_access":{ + "__comment__": "Detects publicly accessible Redshift Clusters.", + "123456789012": ["test-cluster"] + }, + "redshift_encryption":{ + "__comment__": "Detects unencrypted clusters.", + "123456789012": ["test-cluster"] + }, + "ecs_privileged_access":{ + "__comment__": "Detects ECS task definitions which are not enabled logging - task definitions ARNs.", + "1234567890123": ["arn:aws:ecs:us-east-1:1234567890123:task-definition/dev-admin:2993"] + }, + "ecs_logging":{ + "__comment__": "Detects ECS task definitions which are not enabled logging - task definitions ARNs.", + "1234567890123": ["arn:aws:ecs:us-east-1:1234567890123:task-definition/test-admin:2993"] + }, + "ecs_external_image_source":{ + "__comment__": "Detects ECS task definitions which are configured with external image source - task definitions ARNs.", + "1234567890123": ["arn:aws:ecs:us-east-1:1234567890123:task-definition/test-admin:2993"] + }, + "redshift_logging": { + "__comment__": "Detects Redshift clusters which are audit logging is not enabled.", + "123456789012": ["test-cluster"] + }, + "es_domain_logging": { + "__comment__": "Detects Elasticsearch domains which are not enabled logging - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, + "es_unencrypted_domain": { + "__comment__": "Detects Unencrypted Elasticsearch domains - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, + "es_public_access_domain": { + "__comment__": "Detects Unencrypted Elasticsearch publicly accessible domains - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + } } diff --git a/deployment/terraform/modules/identification/identification.tf b/deployment/terraform/modules/identification/identification.tf index 1a5b7b6f..590baade 100755 --- a/deployment/terraform/modules/identification/identification.tf +++ b/deployment/terraform/modules/identification/identification.tf @@ -16,7 +16,16 @@ resource "aws_cloudformation_stack" "identification" { "aws_s3_bucket_object.ami-public-access-issues-identification", "aws_s3_bucket_object.sqs-public-policy-identification", "aws_s3_bucket_object.s3-unencrypted-bucket-issues-identification", - "aws_s3_bucket_object.rds-unencrypted-instance-identification" + "aws_s3_bucket_object.rds-unencrypted-instance-identification", + "aws_s3_bucket_object.ecs-privileged-access-issues-identification", + "aws_s3_bucket_object.ecs-logging-issues-identification", + "aws_s3_bucket_object.ecs-external-image-source-issues-identification", + "aws_s3_bucket_object.redshift-audit-logging-issues-identification", + "aws_s3_bucket_object.redshift-unencrypted-cluster-identification", + "aws_s3_bucket_object.redshift-cluster-public-access-identification", + "aws_s3_bucket_object.elasticsearch-domain-logging-issues-identification", + "aws_s3_bucket_object.elasticsearch-unencrypted-domain-identification", + "aws_s3_bucket_object.elasticsearch-public-access-domain-identification" ] tags = "${var.tags}" @@ -44,7 +53,15 @@ resource "aws_cloudformation_stack" "identification" { SourceIdentificationSQSPublicPolicy = "${aws_s3_bucket_object.sqs-public-policy-identification.id}" SourceIdentificationS3Encryption = "${aws_s3_bucket_object.s3-unencrypted-bucket-issues-identification.id}" SourceIdentificationRDSEncryption = "${aws_s3_bucket_object.rds-unencrypted-instance-identification.id}" + SourceIdentificationECSPrivilegedAccess = "${aws_s3_bucket_object.ecs-privileged-access-issues-identification.id}" + SourceIdentificationECSLogging = "${aws_s3_bucket_object.ecs-logging-issues-identification.id}" + SourceIdentificationECSExternalImageSource = "${aws_s3_bucket_object.ecs-external-image-source-issues-identification.id}" + SourceIdentificationRedshiftLogging = "${aws_s3_bucket_object.redshift-audit-logging-issues-identification.id}" + SourceIdentificationRedshiftClusterEncryption = "${aws_s3_bucket_object.redshift-unencrypted-cluster-identification.id}" + SourceIdentificationRedshiftPublicAccess = "${aws_s3_bucket_object.redshift-cluster-public-access-identification.id}" + SourceIdentificationElasticSearchLogging = "${aws_s3_bucket_object.elasticsearch-domain-logging-issues-identification.id}" + SourceIdentificationElasticSearchEncryption = "${aws_s3_bucket_object.elasticsearch-unencrypted-domain-identification.id}" + SourceIdentificationElasticSearchPublicAccess = "${aws_s3_bucket_object.elasticsearch-public-access-domain-identification.id}" } - template_url = "https://${var.s3bucket}.s3.amazonaws.com/${aws_s3_bucket_object.identification-cfn.id}" } \ No newline at end of file diff --git a/deployment/terraform/modules/identification/sources.tf b/deployment/terraform/modules/identification/sources.tf index e5658577..2d1d5d87 100755 --- a/deployment/terraform/modules/identification/sources.tf +++ b/deployment/terraform/modules/identification/sources.tf @@ -85,6 +85,7 @@ resource "aws_s3_bucket_object" "sqs-public-policy-identification" { key = "lambda/${format("sqs-public-policy-identification-%s.zip", "${md5(file("${path.module}/../../../packages/sqs-public-policy-identification.zip"))}")}" source = "${path.module}/../../../packages/sqs-public-policy-identification.zip" } + resource "aws_s3_bucket_object" "s3-unencrypted-bucket-issues-identification" { bucket = "${var.s3bucket}" key = "lambda/${format("s3-unencrypted-bucket-issues-identification-%s.zip", "${md5(file("${path.module}/../../../packages/s3-unencrypted-bucket-issues-identification.zip"))}")}" @@ -96,3 +97,58 @@ resource "aws_s3_bucket_object" "rds-unencrypted-instance-identification" { key = "lambda/${format("rds-unencrypted-instance-identification-%s.zip", "${md5(file("${path.module}/../../../packages/rds-unencrypted-instance-identification.zip"))}")}" source = "${path.module}/../../../packages/rds-unencrypted-instance-identification.zip" } + +resource "aws_s3_bucket_object" "elasticsearch-unencrypted-domain-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("elasticsearch-unencrypted-domain-identification-%s.zip", "${md5(file("${path.module}/../../../packages/elasticsearch-unencrypted-domain-identification.zip"))}")}" + source = "${path.module}/../../../packages/elasticsearch-unencrypted-domain-identification.zip" +} + +resource "aws_s3_bucket_object" "redshift-cluster-public-access-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("redshift-cluster-public-access-identification-%s.zip", "${md5(file("${path.module}/../../../packages/redshift-cluster-public-access-identification.zip"))}")}" + source = "${path.module}/../../../packages/redshift-cluster-public-access-identification.zip" +} + +resource "aws_s3_bucket_object" "redshift-unencrypted-cluster-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("redshift-unencrypted-cluster-identification-%s.zip", "${md5(file("${path.module}/../../../packages/redshift-unencrypted-cluster-identification.zip"))}")}" + source = "${path.module}/../../../packages/redshift-unencrypted-cluster-identification.zip" +} + +resource "aws_s3_bucket_object" "redshift-audit-logging-issues-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("redshift-audit-logging-issues-identification-%s.zip", "${md5(file("${path.module}/../../../packages/redshift-audit-logging-issues-identification.zip"))}")}" + source = "${path.module}/../../../packages/redshift-audit-logging-issues-identification.zip" +} + +resource "aws_s3_bucket_object" "ecs-privileged-access-issues-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("ecs-privileged-access-issues-identification-%s.zip", "${md5(file("${path.module}/../../../packages/ecs-privileged-access-issues-identification.zip"))}")}" + source = "${path.module}/../../../packages/ecs-privileged-access-issues-identification.zip" +} + +resource "aws_s3_bucket_object" "ecs-logging-issues-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("ecs-logging-issues-identification-%s.zip", "${md5(file("${path.module}/../../../packages/ecs-logging-issues-identification.zip"))}")}" + source = "${path.module}/../../../packages/ecs-logging-issues-identification.zip" +} + +resource "aws_s3_bucket_object" "ecs-external-image-source-issues-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("ecs-external-image-source-issues-identification-%s.zip", "${md5(file("${path.module}/../../../packages/ecs-external-image-source-issues-identification.zip"))}")}" + source = "${path.module}/../../../packages/ecs-external-image-source-issues-identification.zip" +} + +resource "aws_s3_bucket_object" "elasticsearch-domain-logging-issues-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("elasticsearch-domain-logging-issues-identification-%s.zip", "${md5(file("${path.module}/../../../packages/elasticsearch-domain-logging-issues-identification.zip"))}")}" + source = "${path.module}/../../../packages/elasticsearch-domain-logging-issues-identification.zip" +} + +resource "aws_s3_bucket_object" "elasticsearch-public-access-domain-identification" { + bucket = "${var.s3bucket}" + key = "lambda/${format("elasticsearch-public-access-domain-identification-%s.zip", "${md5(file("${path.module}/../../../packages/elasticsearch-public-access-domain-identification.zip"))}")}" + source = "${path.module}/../../../packages/elasticsearch-public-access-domain-identification.zip" +} + diff --git a/docs/_data/sidebars/mydoc_sidebar.yml b/docs/_data/sidebars/mydoc_sidebar.yml index 35adc1bd..d48cf0b7 100644 --- a/docs/_data/sidebars/mydoc_sidebar.yml +++ b/docs/_data/sidebars/mydoc_sidebar.yml @@ -123,3 +123,39 @@ entries: - title: AMI public access url: /playbook13_amis_public_access.html output: web, pdf + + - title: Redshift Unencrypted clusters + url: /playbook15_redshift_unencryption.html + output: web, pdf + + - title: Redshift Publicly Accessible clusters + url: /playbook16_redshift_public_clusters.html + output: web, pdf + + - title: Redshift Audit Logging + url: /playbook17_redshift_audit_logging.html + output: web, pdf + + - title: ECS Logging + url: /playbook18_ecs_logging.html + output: web, pdf + + - title: ECS Privileged Access + url: /playbook19_ecs_privileged_access.html + output: web, pdf + + - title: ECS External Image Source + url: /playbook20_ecs_external_image_source.html + output: web, pdf + + - title: Elasticsearch Domains Unencryption issues + url: /playbook21_elasticsearch_unencryption.html + output: web, pdf + + - title: Elasticsearch Domains Public Access Policy issues + url: /playbook22_elasticsearch_public_access.html + output: web, pdf + + - title: Elasticsearch Domains Logging issues + url: /playbook23_elasticsearch_logging.html + output: web, pdf diff --git a/docs/pages/deployment_cloudformation.md b/docs/pages/deployment_cloudformation.md index 318e6115..99948837 100644 --- a/docs/pages/deployment_cloudformation.md +++ b/docs/pages/deployment_cloudformation.md @@ -99,6 +99,15 @@ You will need to set the following parameters: * **SourceIdentificationS3Encryption**: the relative path to the Lambda package that identifies S3 un-encrypted bucket issues. The default value is **s3-unencrypted-bucket-issues-identification.zip**. * **SourceIdentificationRDSEncryption**: the relative path to the Lambda package that identifies RDS unencrypted instances. The default value is **rds-unencrypted-instance-identification.zip**. * **SourceIdentificationAMIPublicAccess**: the relative path to the Lambda package that identifies Public AMIs. The default value is **ami-public-acess-issues-identification.zip**. +* **SourceIdentificationECSPrivilegedAccess**: the relative path to the Lambda package that identifies ECS privileged access issues. The default value is **ecs-privileged-access-issues-identification.zip**. +* **SourceIdentificationECSExternalImageSource**: the relative path to the Lambda package that identifies ECS external image source issues. The default value is **ecs-external-image-source-issues-identification.zip**. +* **SourceIdentificationECSLogging**: the relative path to the Lambda package that identifies ECS logging issues. The default value is **ecs-logging-issues-identification.zip**. +* **SourceIdentificationRedshiftLogging**: the relative path to the Lambda package that identifies audit logging redshift issues. The default value is **redshift-audit-logging-issues-identification.zip**. +* **SourceIdentificationRedshiftClusterEncryption**: the relative path to the Lambda package that identifies unencrypted redshift cluster issues. The default value is **redshift-unencrypted-cluster-identification.zip**. +* **SourceIdentificationRedshiftPublicAccess**: the relative path to the Lambda package that identifies publicly accessibly redshift cluster issues. The default value is **redshift-cluster-public-access-identification.zip**. +* **SourceIdentificationElasticSearchEncryption**: the relative path to the Lambda package that identifies Elasticsearch domain encryption issues. The default value is **elasticsearch-unencrypted-domain-identification.zip**. +* **SourceIdentificationElasticSearchLogging**: the relative path to the Lambda package that identifies Elasticsearch domain logging issues. The default value is **elasticsearch-domain-logging-issues-identification.zip**. +* **SourceIdentificationElasticSearchPublicAccess**: the relative path to the Lambda package that identifies Elasticsearch domain public access issues. The default value is **elasticsearch-public-access-domain-identification.zip**. **VPC config (optional)**: * **LambdaSubnets**: comma-separated list, without spaces, of subnet IDs in your VPC to run identification lambdas in. diff --git a/docs/pages/editconfig.md b/docs/pages/editconfig.md index cfd5eb9d..58909380 100644 --- a/docs/pages/editconfig.md +++ b/docs/pages/editconfig.md @@ -400,3 +400,117 @@ Parameters: * **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; * **remediation**: defines whether Dow Jones Hammer will automatically remediate the detected issue. The default value is `false`; * **remediation_retention_period**: the amount of days that should pass between the detection of an issue and its automatic remediation by Dow Jones Hammer. The default value is `0`. + +### 2.15. Redshift unencrypted cluster issues + +This section describes how to detect whether you have unencrypted redshift cluster issues or not. Refer to [issue-specific playbook](playbook15_redshift_unencryption.html) for further details. + +Edit the **redshift_encryption** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-redshift-unencrypted`. +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; +* **remediation**: defines whether Dow Jones Hammer will automatically remediate the detected issue. The default value is `false`; +* **remediation_retention_period**: the amount of days that should pass between the detection of an issue and its automatic remediation by Dow Jones Hammer. The default value is `0`. + +### 2.16. Redshift publicly accessibly cluster issues. + +This section describes how to detect whether you have publicly accessible redshift cluster issues or not. Refer to [issue-specific playbook](playbook16_redshift_public_clusters.html) for further details. + +Edit the **redshift_public_access** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-redshift-public-access`. +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; +* **remediation**: defines whether Dow Jones Hammer will automatically remediate the detected issue. The default value is `false`; +* **remediation_retention_period**: the amount of days that should pass between the detection of an issue and its automatic remediation by Dow Jones Hammer. The default value is `0`. + +### 2.17. Redshift logging issues + +This section describes how to detect whether you have redshift logging enabled or not. Refer to [issue-specific playbook](playbook17_redshift_audit_logging.html) for further details. + +Edit the **redshift_logging** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-redshift-logging`. +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false + +### 2.18. ECS logging issues + +This section describes how to detect whether you have ecs logging enabled or not. Refer to [issue-specific playbook](playbook18_ecs_logging.html) for further details. + +Edit the **ecs_logging** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-ecs-logging`. +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; + +### 2.19. ECS Privileged Access issues. + +This section describes how to detect whether you have ECS privileged accesss enabled or not. Refer to [issue-specific playbook](playbook19_ecs_privileged_access.html) for further details. + +Edit the **ecs_privileged_access** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-ecs-privileged-access`. +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; + + +### 2.20. ECS external image source issues + +This section describes how to detect whether you have ECS image source is external or internal. Refer to [issue-specific playbook](playbook20_ecs_external_image_source.html) for further details. + +Edit the **ecs_external_image_source** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-ecs-external-image-source`. +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; + +### 2.21. Elasticsearch Domain Encryption Issues + +This section describes how to detect whether you have Elasticesearch Domains encrypted or not. Refer to [issue-specific playbook](playbook21_elasticsearch_unencryption.html) for further details. + +Edit the **es_unencrypted_domain** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-es-unencrypted-domain`. +* **accounts**: *optional* comma-separated list of accounts to check and report for issue in square brackets. Use this key to override accounts from **aws.accounts** in [config.json](#11-master-aws-account-settings); +* **ignore_accounts**: *optional* comma-separated list of accounts to ignore during check. Use this key to exclude accounts from **aws.accounts** in [config.json](#11-master-aws-account-settings); +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; + +### 2.22. Elasticsearch Domain Public Access Issues + +This section describes how to detect whether you have Elasticesearch Domains has public access policy. Refer to [issue-specific playbook](playbook22_elasticsearch_public_access.html) for further details. + +Edit the **es_public_access_domain** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `es-public-access-domain`. +* **accounts**: *optional* comma-separated list of accounts to check and report for issue in square brackets. Use this key to override accounts from **aws.accounts** in [config.json](#11-master-aws-account-settings); +* **ignore_accounts**: *optional* comma-separated list of accounts to ignore during check. Use this key to exclude accounts from **aws.accounts** in [config.json](#11-master-aws-account-settings); +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; +* **remediation**: defines whether Dow Jones Hammer will automatically remediate the detected issue. The default value is `false`; +* **remediation_retention_period**: the amount of days that should pass between the detection of an issue and its automatic remediation by Dow Jones Hammer. The default value is `0`. + +### 2.23. Elasticsearch Domain Logging Issues + +This section describes how to detect whether you have Elasticesearch Domains logging enabled or not. Refer to [issue-specific playbook](playbook23_elasticsearch_logging.html) for further details. + +Edit the **es_domain_logging** section of the `config.json` file to configure the handling of this issue. + +Parameters: +* **enabled**: enables/disables issue identification. The default value is `true`; +* **ddb.table_name**: the name of the DynamoDB table where Dow Jones Hammer will put detection results. The default value is `hammer-es-domain-logging`. +* **accounts**: *optional* comma-separated list of accounts to check and report for issue in square brackets. Use this key to override accounts from **aws.accounts** in [config.json](#11-master-aws-account-settings); +* **ignore_accounts**: *optional* comma-separated list of accounts to ignore during check. Use this key to exclude accounts from **aws.accounts** in [config.json](#11-master-aws-account-settings); +* **reporting**: defines whether Dow Jones Hammer will report detected issues to JIRA/Slack. The default value is `false`; +* **remediation**: defines whether Dow Jones Hammer will automatically remediate the detected issue. The default value is `false`; +* **remediation_retention_period**: the amount of days that should pass between the detection of an issue and its automatic remediation by Dow Jones Hammer. The default value is `0`. diff --git a/docs/pages/features.md b/docs/pages/features.md index 65e95d51..26fb80ce 100644 --- a/docs/pages/features.md +++ b/docs/pages/features.md @@ -22,5 +22,15 @@ Dow Jones Hammer can identify and report the following issues: |[S3 Unencrypted Buckets](playbook11_s3_unencryption.html) |Detects not encrypted at reset S3 buckets |Any of S3 bucket is not encrypted at rest | |[RDS Unencrypted instances](playbook12_rds_unencryption.html) |Detects not encrypted at rest RDS instances |Any one of RDS instances is not encrypted at reset | |[AMIs public access](playbook13_amis_public_access.html) |Detects publicly accessible AMIs |Any one of AMI is worldwide accessible | +|[Redshift Unencrypted Clusters](playbook15_redshift_unencryption.html) |Detects Redshift unencrypted cluster issues |Any one of Redshift cluster is not encrypted at rest | +|[Redshift Public Access Clusters](playbook16_redshift_public_clusters.html) |Detects Redshift publicly accessible cluster isues |Any one of Redshift cluster publicly accessible | +|[Redshift Logging Issues](playbook17_redshift_audit_logging.html) |Detects Redshift logging issues |Any one of Redshift cluster logging is not enabled | +|[ECS Logging](playbook18_ecs_logging.html) |Detects ECS task definition's logging issues |Any one of ECS task definition's logging enabled or not | +|[ECS Privileged Access](playbook19_ecs_privileged_access.html) |Detects ECS task definition's privileged access issues |Any one of ECS task definition have privileged access enabled or not | +|[ECS External Image Source](playbook20_ecs_external_image_source.html) |Detects ECS task definitions image source issues |Any one of ECS image source is external or internal | +|[Elasticsearch Domain Encryption Issues](playbook21_elasticsearch_unencryption.html) |Detects Elasticsearch domains encryption issues |Any one of Elasticsearch Domain unencryption issue | +|[Elasticsearch Domain Public Access Issues](playbook22_elasticsearch_public_access.html) |Detects Elasticsearch domain public access issues |Any one of Elasticsearch Domain public access issue | +|[Elasticsearch Domain Logging Issues](playbook23_elasticsearch_logging.html) |Detects Elasticsearch domains logging issues |Any one of Elasticsearch Domain logging issue | + +Dow Jones Hammer can perform remediation for all issues [except](remediation_backup_rollback.html#1-overview) **EBS Unencrypted volumes**, **CloudTrail Logging Issues**, **RDS Unencrypted instances** **ECS Logging**, **ECS Privileged Access**, **ECS External Image Source**, **Redshift Audit Logging** and **Elasticsearch Domain Encryption Issues**. -Dow Jones Hammer can perform remediation for all issues [except](remediation_backup_rollback.html#1-overview) **EBS Unencrypted volumes**, **CloudTrail Logging Issues** and **RDS Unencrypted instances**. \ No newline at end of file diff --git a/docs/pages/playbook10_sqs_public_policy.md b/docs/pages/playbook10_sqs_public_policy.md index 304bf1db..85a173b7 100644 --- a/docs/pages/playbook10_sqs_public_policy.md +++ b/docs/pages/playbook10_sqs_public_policy.md @@ -88,6 +88,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-sqs-public-policy-lambda`| Sample **config.json** section: ``` @@ -96,7 +97,8 @@ Sample **config.json** section: "ddb.table_name": "hammer-sqs-public-access", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-sqs-public-policy-lambda" }, ``` diff --git a/docs/pages/playbook11_s3_unencryption.md b/docs/pages/playbook11_s3_unencryption.md index 15b2c54b..1d168f90 100644 --- a/docs/pages/playbook11_s3_unencryption.md +++ b/docs/pages/playbook11_s3_unencryption.md @@ -83,6 +83,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-s3-encryption-lambda`| Sample **config.json** section: ``` @@ -91,7 +92,8 @@ Sample **config.json** section: "ddb.table_name": "hammer-s3-unencrypted", "reporting": true, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-s3-encryption-lambda" } ``` diff --git a/docs/pages/playbook12_rds_unencryption.md b/docs/pages/playbook12_rds_unencryption.md index aceb9f10..87dec997 100644 --- a/docs/pages/playbook12_rds_unencryption.md +++ b/docs/pages/playbook12_rds_unencryption.md @@ -58,13 +58,15 @@ To identify and report issues of this type, you should add the following paramet |`enabled` |Toggles issue detection for this issue |`true`| |`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-rds-unencrypted` | |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-rds-encryption-lambda`| Sample **config.json** section: ``` "rds_encryption": { "enabled": true, "ddb.table_name": "hammer-rds-unencrypted", - "reporting": true + "reporting": true, + "topic_name": "hammer-describe-rds-encryption-lambda" } ``` diff --git a/docs/pages/playbook13_amis_public_access.md b/docs/pages/playbook13_amis_public_access.md index a3e6c53c..22653eb2 100644 --- a/docs/pages/playbook13_amis_public_access.md +++ b/docs/pages/playbook13_amis_public_access.md @@ -58,6 +58,7 @@ To identify and report issues of this type, you should add the following paramet |`enabled` |Toggles issue detection for this issue |`true`| |`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-public-amis` | |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-ami-public-access-lambda`| Sample **config.json** section: ``` @@ -66,7 +67,8 @@ Sample **config.json** section: "ddb.table_name": "djif-hammer-public-amis", "reporting": true, "remediation": false, - "remediation_retention_period": 21 + "remediation_retention_period": 21, + "topic_name": "hammer-describe-ami-public-access-lambda" } ``` diff --git a/docs/pages/playbook15_redshift_unencryption.md b/docs/pages/playbook15_redshift_unencryption.md new file mode 100644 index 00000000..91db0a4b --- /dev/null +++ b/docs/pages/playbook15_redshift_unencryption.md @@ -0,0 +1,201 @@ +--- +title: Redshift unencrypted cluster issues +keywords: playbook15 +sidebar: mydoc_sidebar +permalink: playbook15_redshift_unencryption.html +--- + +# Playbook 15: Redshift unencrypted cluster issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect Redshift unencrypted cluster issues. + +## 1. Issue Identification + +Dow Jones Hammer identifies those Redshift clusters 'Encrypted' parameters. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/redshift-unencrypted-cluster-identification/initiate_to_desc_redshift_encryption.py`| +|Identification|`hammer/identification/lambdas/redshift-unencrypted-cluster-identification/describe_redshift_encryption.py`| + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining Redshift cluster encryption status. In case the cluster is unencrypted **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_redshift_unencrypted_cluster_issue_tickets.py +``` + +## 3. Issue Remediation + +### 3.1 Automatic + +To reduce the workload of your DevOps engineers and mitigate the threats stemming from this issue, you can configure automatic remediation of issues. It means that in case Dow Jones Hammer has detected and reported an issue, but the assignee of the report has not remediated the issue within a timeframe specified in the configuration, the Dow Jones Hammer remediation job will encrypt Redshift cluster eliminate this vulnerability. + +This Python module implements the issue remediation functionality: +``` +hammer/reporting-remediation/remediation/clean_redshift_cluster_unencrypted.py + +### 3.2 Manual + +To retain full control on the remediation functionality you can disable automatic remediation in [config.json] and launch it manually: +1. Login to Dow Jones Hammer reporting and remediation EC2 via SSH with **centos** user and ssh key you created during [deployment](configuredeploy_overview.html#25-create-ec2-key-pair-for-hammer): `ssh -l centos -i ` +2. Become **root** user: `sudo su -` +3. Change directory to Dow Jones Hammer sources: `cd /hammer-correlation-engine` +4. Launch Dow Jones Hammer remediation script: `python3.6 -m remediation.clean_redshift_cluster_unencrypted` +5. Confirm or refuse remediation of each issue separately + +## 4. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 4.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **redshift_encryption** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-redshift-unencrypted` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| +|`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-redshift-cluster-encryption-lambda`| + + +Sample **config.json** section: +``` +"redshift_encryption": { + "enabled": true, + "ddb.table_name": "hammer-redshift-unencrypted", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-redshift-cluster-encryption-lambda" + } +``` + +### 4.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific Redshift clusters. To configure such exceptions, you should edit the **redshift_encryption** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Redshift cluster ids(s)| + +Sample **whitelist.json** section: +``` +"redshift_encryption": { + "123456789012": ["redshift_id1", "redshift_id2"] +} +``` + +### 4.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 5. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 5.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/initiate-redshift-encryption`| +|Identification |`/aws/lambda/describe-redshift-encryption`| + + +### 5.2. Issue Reporting/Remediation Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_redshift_unencrypted_cluster_issue_tickets`| +|Remediation|`remediation.clean_redshift_cluster_unencrypted` | + +### 5.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 5.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 6. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`id` |string|redshift id |`redshift-id` | +|`tags` |map |Tags associated with Redshift id |`{"Name": "TestKey", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook16_redshift_public_clusters.md b/docs/pages/playbook16_redshift_public_clusters.md new file mode 100644 index 00000000..19ee830a --- /dev/null +++ b/docs/pages/playbook16_redshift_public_clusters.md @@ -0,0 +1,181 @@ +--- +title: Redshift publicly accessible cluster issues +keywords: playbook16 +sidebar: mydoc_sidebar +permalink: playbook16_redshift_public_clusters.html +--- + +# Playbook 16: Redshift publicly accessible cluster issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect Redshift public accessible cluster issues. + +## 1. Issue Identification + +Dow Jones Hammer identifies those Redshift clusters ```PubliclyAccessible``` parameters. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/redshift-cluster-public-access-identification/initiate_to_desc_redshift_cluster_public_access.py`| +|Identification|`hammer/identification/lambdas/redshift-cluster-public-access-identification/describe_redshift_cluster_public_access.py`| + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining Redshift cluster publicly accessible status. In case the cluster is publicly accessible (true) **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_redshift_public_access_issue_tickets.py +``` + + +## 3. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 3.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **redshift_public_access** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-redshift-public-access` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-redshift-public-access-lambda`| + + +Sample **config.json** section: +``` +""" +"redshift_public_access": { + "enabled": true, + "ddb.table_name": "hammer-redshift-public-access", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-redshift-public-access-lambda" + } + ``` + +### 3.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific Redshift clusters. To configure such exceptions, you should edit the **redshift_public_access** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Redshift cluster ids(s)| + +Sample **whitelist.json** section: +``` +"redshift_public_access": { + "123456789012": ["redshift_id1", "redshift_id2"] +} +``` + +### 3.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 4. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 4.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/initiate-redshift-public-access`| +|Identification |`/aws/lambda/describe-redshift-public-access`| + +### 4.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_redshift_public_access_issue_tickets`| + + +### 4.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 4.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 5. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`id` |string|redshift id |`redshift-id` | +|`tags` |map |Tags associated with Redshift id |`{"Name": "TestKey", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook17_redshift_audit_logging.md b/docs/pages/playbook17_redshift_audit_logging.md new file mode 100644 index 00000000..c1611ec5 --- /dev/null +++ b/docs/pages/playbook17_redshift_audit_logging.md @@ -0,0 +1,180 @@ +--- +title: Redshift audit logging issues +keywords: playbook17 +sidebar: mydoc_sidebar +permalink: playbook17_redshift_audit_logging.html +--- + +# Playbook 17: Redshift audit logging issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect Redshift audit logging issues. + +## 1. Issue Identification + +Dow Jones Hammer identifies those Redshift clusters logging enabled or not. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/redshift-audit-logging-issues-identification/initiate_to_desc_redshift_logging_issues.py`| +|Identification|`hammer/identification/lambdas/redshift-audit-logging-issues-identification/describe_redshift_logging_issues.py`| + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining Redshift cluster logging status. In case the cluster logging is not enable **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_redshift_logging_issue_tickets.py +``` + + +## 3. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 3.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **redshift_logging** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-redshift-logging` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-redshift-logging-lambda`| + +Sample **config.json** section: +``` +""" +"redshift_logging": { + "enabled": true, + "ddb.table_name": "hammer-redshift-logging", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-redshift-logging-lambda" + } +``` + +### 3.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific Redshift clusters. To configure such exceptions, you should edit the **redshift_logging** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Redshift cluster ids(s)| + +Sample **whitelist.json** section: +``` +"redshift_logging": { + "123456789012": ["redshift_id1", "redshift_id2"] +} +``` + +### 3.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 4. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 4.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/initiate-redshift-logging`| +|Identification |`/aws/lambda/describe-redshift-logging`| + +### 4.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_redshift_logging_issue_tickets`| + + +### 4.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 4.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 5. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`id` |string|redshift id |`redshift-id` | +|`tags` |map |Tags associated with Redshift id |`{"Name": "TestKey", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook18_ecs_logging.md b/docs/pages/playbook18_ecs_logging.md new file mode 100644 index 00000000..1340f780 --- /dev/null +++ b/docs/pages/playbook18_ecs_logging.md @@ -0,0 +1,179 @@ +--- +title: ECS logging issues +keywords: playbook18 +sidebar: mydoc_sidebar +permalink: playbook18_ecs_logging.html +--- + +# Playbook 18: ECS logging issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect ECS logging issues. + +## 1. Issue Identification + +Dow Jones Hammer identifies those ECS logging enabled or not. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/ecs-logging-issues-identification/initiate_to_desc_ecs_logging_issues.py`| +|Identification|`hammer/identification/lambdas/ecs-logging-issues-identification/describe_ecs_logging_issues.py`| + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining ECS logging status. In case the logging is not enable **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_ecs_logging_issue_tickets.py +``` + +## 3. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 3.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **ecs_logging** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-ecs-logging` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`true`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-ecs-logging-lambda`| + +Sample **config.json** section: +``` +""" +"ecs_logging": { + "enabled": true, + "ddb.table_name": "hammer-ecs-logging", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-ecs-logging-lambda" + } +``` + +### 3.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific ECS task definitions. To configure such exceptions, you should edit the **ecs_logging** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|ECS task definition ids(s)| + +Sample **whitelist.json** section: +``` +"ecs_logging": { + "123456789012": ["task_definition_id1", "task_definition2"] +} +``` + +### 3.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 4. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 4.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/initiate-ecs-logging`| +|Identification |`/aws/lambda/describe-ecs-logging`| + +### 4.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_ecs_logging_issue_tickets`| + + +### 4.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 4.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 5. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`id` |string|ecs task definition id |`task-definition-id` | +|`tags` |map |Tags associated with ECS task |`{"Name": "TestKey", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook19_ecs_privileged_access.md b/docs/pages/playbook19_ecs_privileged_access.md new file mode 100644 index 00000000..eca6ba36 --- /dev/null +++ b/docs/pages/playbook19_ecs_privileged_access.md @@ -0,0 +1,180 @@ +--- +title: ECS Privileged Access issues +keywords: playbook19 +sidebar: mydoc_sidebar +permalink: playbook19_ecs_privileged_access.html +--- + +# Playbook 19: ECS Privileged Access issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect ECS privileged access issues. + +## 1. Issue Identification + +Dow Jones Hammer identifies those ECS privileged access is enabled or not. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/ecs-privileged-access-issues-identification/initiate_to_desc_ecs_privileged_access_issues.py`| +|Identification|`hammer/identification/lambdas/ecs-privileged-access-issues-identification/describe_ecs_privileged_issues.py`| + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining ECS privileged access issues. In case the privileged access is enable **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_ecs_privileged_access_issue_tickets.py +``` + + +## 3. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 3.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **ecs_privileged_access** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-ecs-privileged-access` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`true`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-ecs-privileged-access-lambda`| + +Sample **config.json** section: +``` +""" +"ecs_privileged_access": { + "enabled": true, + "ddb.table_name": "hammer-ecs-privileged-access", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-ecs-privileged-access-lambda" + } +``` + +### 3.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific ECS task definitions. To configure such exceptions, you should edit the **ecs_privileged_access** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|ECS task definition ids(s)| + +Sample **whitelist.json** section: +``` +"ecs_privileged_access": { + "123456789012": ["task_definition_arn1", "task_definition_arn2"] +} +``` + +### 3.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 4. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 4.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/initiate-ecs-privileged-access`| +|Identification |`/aws/lambda/describe-privileged-access`| + +### 4.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_ecs_privileged_access_issue_tickets`| + + +### 4.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 4.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 5. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`id` |string|ecs task definition id |`ecs-task_definition-id` | +|`tags` |map |Tags associated with ECS task definition id |`{"Name": "TestKey", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook1_s3_public_buckets_acl.md b/docs/pages/playbook1_s3_public_buckets_acl.md index d3d159c8..78fe82b1 100644 --- a/docs/pages/playbook1_s3_public_buckets_acl.md +++ b/docs/pages/playbook1_s3_public_buckets_acl.md @@ -87,6 +87,8 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-s3-acl-lambda`| + Sample **config.json** section: ``` @@ -95,7 +97,8 @@ Sample **config.json** section: "ddb.table_name": "hammer-s3-public-bucket-acl", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-s3-acl-lambda", } ``` diff --git a/docs/pages/playbook20_ecs_external_image_source.md b/docs/pages/playbook20_ecs_external_image_source.md new file mode 100644 index 00000000..b2fe165a --- /dev/null +++ b/docs/pages/playbook20_ecs_external_image_source.md @@ -0,0 +1,179 @@ +--- +title: ECS Image Source issues +keywords: playbook20 +sidebar: mydoc_sidebar +permalink: playbook20_ecs_external_image_source.html +--- + +# Playbook 20: ECS Image Source issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect ECS image source is external or internal based on configured image known sources. + +## 1. Issue Identification + +Dow Jones Hammer identifies those ECS image source is external or internal. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/ecs-external-image-source-issues-identification/initiate_to_desc_ecs_external_image_source_issues.py`| +|Identification|`hammer/identification/lambdas/ecs-external-image-source-issues-identification/describe_external_image_source_issues.py`| + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining ECS image source status. In case the ECS image source is external or internal **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_ecs_external_image_source_issue_tickets.py +``` + + +## 3. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 3.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **ecs_external_image_source** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-ecs-external-image-source` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`true`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-ecs-external-image-source-lambda`| + +Sample **config.json** section: +``` +""" +"ecs_external_image_source": { + "enabled": true, + "ddb.table_name": "hammer-ecs-external-image-source", + "reporting": true, + "safe_image_sources": ["amazonaws", "artifactory"], + "topic_name": "hammer-describe-ecs-external-image-source-lambda" + } +``` + +### 3.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific ECS task definitions. To configure such exceptions, you should edit the **ecs_external_image_source** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Task definition arn(s)| + +Sample **whitelist.json** section: +``` +"ecs_external_image_source": { + "123456789012": ["task_definition_id1", "task_definition2"] +} +``` + +### 3.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 4. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 4.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/initiate-ecs-external-image-source`| +|Identification |`/aws/lambda/describe-ecs-external-image-source`| + +### 4.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_ecs_external_image_source_issue_tickets`| + + +### 4.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 4.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 5. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`id` |string|ecs task definition id |`task-definition-id` | +|`tags` |map |Tags associated with ECS task |`{"Name": "TestKey", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook21_elasticsearch_unencryption.md b/docs/pages/playbook21_elasticsearch_unencryption.md new file mode 100644 index 00000000..0621331c --- /dev/null +++ b/docs/pages/playbook21_elasticsearch_unencryption.md @@ -0,0 +1,179 @@ +--- +title: Elasticsearch unencrypted instances +keywords: playbook21 +sidebar: mydoc_sidebar +permalink: playbook21_elasticsearch_unencryption.html +--- + +# Playbook 21: Elasticsearch unencrypted instances + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect Elasticsearch domains that are not encrypted (either at dataatrest or node-node encryption). + +## 1. Issue Identification + +Dow Jones Hammer identifies those Elasticsearch domains for which ```StorageEncrypted``` parameter value is ```false```. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/initiate_to_desc_elasticsearch_unencrypted_domains.py`| +|Identification|`hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/describe_elasticsearch_unencrypted_domains.py` | + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining `owner` tag on affected RDS instance. In case when such tag **exists** and is **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_elasticsearch_unencrypted_issue_tickets.py +``` + + +## 3. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 3.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **rds_encryption** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-es-unencrypted-domain` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-es-encryption-lambda`| + +Sample **config.json** section: +``` +"es_unencrypted_domain": { + "enabled": true, + "ddb.table_name": "djif-hammer-es-unencrypted-domain", + "reporting": true, + "topic_name": "hammer-describe-es-encryption-lambda" + }, +``` + +### 3.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific RDS instances. To configure such exceptions, you should edit the **rds_encryption** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Elasticsearch Domain Names(s)| + +Sample **whitelist.json** section: +``` +"es_unencrypted_domain": { + "__comment__": "Detects Unencrypted Elasticsearch domains - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, +``` + +### 3.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 4. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 4.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/hammer-initiate-rds-encryption`| +|Identification |`/aws/lambda/hammer-describe-rds-encryption`| + +### 4.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_rds_unencrypted_instance_issue_tickets`| + + +### 4.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 4.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 5. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`name` |string|RDS instance name |`test-rds-instances` | +|`engine` |string|Name of the database engine |`mysql` | +|`tags` |map |Tags associated with RDS instance |`{"Name": "TestInstance", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook22_elasticsearch_public_access.md b/docs/pages/playbook22_elasticsearch_public_access.md new file mode 100644 index 00000000..b1e3b136 --- /dev/null +++ b/docs/pages/playbook22_elasticsearch_public_access.md @@ -0,0 +1,203 @@ +--- +title: Elasticsearch Domain public access +keywords: playbook22 +sidebar: mydoc_sidebar +permalink: playbook22_elasticsearch_public_access.html +--- + +# Playbook 22: Elasticsearch publicly accessible domains + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect Elasticsearch domains that are publicly accessible + +## 1. Issue Identification + +Dow Jones Hammer identifies those Elasticsearch domains for public access policy attached. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/elasticsearch-public-access-domain-identification/initiate_to_desc_elasticsearch_public_access_domains.py`| +|Identification|`hammer/identification/lambdas/elasticsearch-public-access-domain-identification/describe_elasticsearch_public_access_domains.py` | + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining `owner` tag on affected Elasticsearch domains. In case when such tag **exists** and is **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_elasticsearch_public_access_issue_tickets.py +``` + +## 3. Issue Remediation + +### 3.1 Automatic + +To reduce the workload of your DevOps engineers and mitigate the threats stemming from this issue, you can configure automatic remediation of issues. It means that in case Dow Jones Hammer has detected and reported an issue, but the assignee of the report has not remediated the issue within a timeframe specified in the configuration, the Dow Jones Hammer remediation job will adjust Elasticsearch Domain policy to eliminate this vulnerability. + +In this specific case, Dow Jones Hammer restricts public statement by adding (or changing) `IpAddress` condition value that allow access only for IP addresses defined in [RFC 1918 - Address Allocation for Private Internets](https://tools.ietf.org/html/rfc1918). + +This Python module implements the issue remediation functionality: +``` +hammer/reporting-remediation/remediation/clean_elasticsearch_policy_permissions.py +``` + +### 3.2 Manual + +To retain full control on the remediation functionality you can disable automatic remediation in [config.json](#41-the-configjson-file) and launch it manually: +1. Login to Dow Jones Hammer reporting and remediation EC2 via SSH with **centos** user and ssh key you created during [deployment](configuredeploy_overview.html#25-create-ec2-key-pair-for-hammer): `ssh -l centos -i ` +2. Become **root** user: `sudo su -` +3. Change directory to Dow Jones Hammer sources: `cd /hammer-correlation-engine` +4. Launch Dow Jones Hammer remediation script: `python3.6 -m remediation.clean_elasticsearch_policy_permissions` +5. Confirm or refuse remediation of each issue separately + + +## 4. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 4.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **es_public_access_domain** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-es-public-access-domain` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-es-public-access-lambda`| + +Sample **config.json** section: +``` +"es_public_access_domain": { + "enabled": true, + "ddb.table_name": "hammer-es-public-access-domain", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-es-public-access-lambda" + },``` + +### 4.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific Elasticsearch Domains. To configure such exceptions, you should edit the **es_public_access_domain** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Elasticsearch Domain Names(s)| + +Sample **whitelist.json** section: +``` +"es_public_access_domain": { + "__comment__": "Detects publicly accessible Elasticsearch domains - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, +``` + +### 4.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 5. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 5.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/hammer-initiate-elasticsearch-public-access`| +|Identification |`/aws/lambda/hammer-describe-elasticsearch-public-access`| + +### 5.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_elasticsearch_public_access_issue_tickets`| +|Remediation |`remediation.clean_elasticsearch_policy_permissions`| + + +### 5.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 5.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 6. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`name` |string|Elasticsearch domain name |`test-domain` | +|`arn` |string|Elasticsearch Domain Arn |`arn:aws:es:us-east-2:1234567890123:domain/test-domain` | +|`tags` |map |Tags associated with Domain |`{"Name": "TestDomain", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook23_elasticsearch_logging.md b/docs/pages/playbook23_elasticsearch_logging.md new file mode 100644 index 00000000..c4913671 --- /dev/null +++ b/docs/pages/playbook23_elasticsearch_logging.md @@ -0,0 +1,202 @@ +--- +title: Elasticsearch logging issues +keywords: playbook23 +sidebar: mydoc_sidebar +permalink: playbook23_elasticsearch_logging.html +--- + +# Playbook 23: Elasticsearch logging issues + +## Introduction + +This playbook describes how to configure Dow Jones Hammer to detect Elasticsearch domains logging enabled or not. + +## 1. Issue Identification + +Dow Jones Hammer identifies those Elasticsearch domains logging enabled or not.. + +When Dow Jones Hammer detects an issue, it writes the issue to the designated DynamoDB table. + +According to the [Dow Jones Hammer architecture](/index.html), the issue identification functionality uses two Lambda functions. +The table lists the Python modules that implement this functionality: + +|Designation |Path | +|--------------|:--------------------:| +|Initialization|`hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/initiate_to_desc_elasticsearch_domains_logging_issues.py`| +|Identification|`hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/describe_elasticsearch_domains_logging_issues.py` | + +## 2. Issue Reporting + +You can configure automatic reporting of cases when Dow Jones Hammer identifies an issue of this type. Dow Jones Hammer supports integration with [JIRA](https://www.atlassian.com/software/jira) and [Slack](https://slack.com/). +These types of reporting are independent from one another and you can turn them on/off in the Dow Jones Hammer configuration. + +Thus, in case you have turned on the reporting functionality for this issue and configured corresponding integrations, Dow Jones Hammer, as [defined in the configuration](#43-the-ticket_ownersjson-file), can: +* raise a JIRA ticket and assign it to a specific person in your organization; +* send the issue notification to the Slack channel or directly to a Slack user. + +Additionally Dow Jones Hammer tries to detect person to report issue to by examining `owner` tag on affected Elasticsearch domain. In case when such tag **exists** and is **valid JIRA/Slack user**: +* for JIRA: `jira_owner` parameter from [ticket_owners.json](#43-the-ticket_ownersjson-file) **is ignored** and discovered `owner` **is used instead** as a JIRA assignee; +* for Slack: discovered `owner` **is used in addition to** `slack_owner` value from [ticket_owners.json](#43-the-ticket_ownersjson-file). + +This Python module implements the issue reporting functionality: +``` +hammer/reporting-remediation/reporting/create_elasticsearch_domain_logging_issue_tickets.py +``` + +## 3. Issue Remediation + +### 3.1 Automatic + +To reduce the workload of your DevOps engineers and mitigate the threats stemming from this issue, you can configure automatic remediation of issues. It means that in case Dow Jones Hammer has detected and reported an issue, but the assignee of the report has not remediated the issue within a timeframe specified in the configuration, the Dow Jones Hammer remediation job will add/adjust logging for Elasticsearch Domain to eliminate this vulnerability. + +This Python module implements the issue remediation functionality: +``` +hammer/reporting-remediation/remediation/clean_elasticsearch_domain_logging.py +``` + +### 3.2 Manual + +To retain full control on the remediation functionality you can disable automatic remediation in [config.json](#41-the-configjson-file) and launch it manually: +1. Login to Dow Jones Hammer reporting and remediation EC2 via SSH with **centos** user and ssh key you created during [deployment](configuredeploy_overview.html#25-create-ec2-key-pair-for-hammer): `ssh -l centos -i ` +2. Become **root** user: `sudo su -` +3. Change directory to Dow Jones Hammer sources: `cd /hammer-correlation-engine` +4. Launch Dow Jones Hammer remediation script: `python3.6 -m remediation.clean_elasticsearch_domain_logging` +5. Confirm or refuse remediation of each issue separately + +## 4. Setup Instructions For This Issue + +To configure the detection, reporting, you should edit the following sections of the Dow Jones Hammer configuration files: + +### 4.1. The config.json File + +The **config.json** file is the main configuration file for Dow Jones Hammer that is available at `deployment/terraform/accounts/sample/config/config.json`. +To identify and report issues of this type, you should add the following parameters in the **es_domain_logging** section of the **config.json** file: + +|Parameter Name |Description | Default Value| +|------------------------------|---------------------------------------|:------------:| +|`enabled` |Toggles issue detection for this issue |`true`| +|`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type| `hammer-es-domain-logging` | +|`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`remediation` |Toggle Dow Jones Hammer remediation functionality for this issue type |`false`| +|`remediation_retention_period` |Toggle Dow Jones Hammer remediation retention period details for this issue type |`21`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-es-logging-lambda`| + +Sample **config.json** section: +``` +"es_domain_logging": { + "enabled": true, + "ddb.table_name": "hammer-es-domain-logging", + "reporting": true, + "remediation": false, + "remediation_retention_period": 21, + "topic_name": "hammer-describe-es-logging-lambda" + } +``` + +### 4.2. The whitelist.json File + +You can define exceptions to the general automatic remediation settings for specific Elasticsearch domains. To configure such exceptions, you should edit the **es_domain_logging** section of the **whitelist.json** configuration file as follows: + +|Parameter Key | Parameter Value(s)| +|:------------:|:-----------------:| +|AWS Account ID|Elasticsearch Domain Names(s)| + +Sample **whitelist.json** section: +``` +"es_domain_logging": { + "__comment__": "Detects Elasticsearch domains which are not enabled logging - domain ARNs.", + "1234567890123": ["arn:aws:es:us-east-2:1234567890123:domain/new-domain"] + }, +``` + +### 4.3. The ticket_owners.json File + +You should use the **ticket_owners.json** file to configure the integration of Dow Jones Hammer with JIRA and/or Slack for the issue reporting purposes. + +You can configure these parameters for specific AWS accounts and globally. Account-specific settings precede the global settings in the **ticket_owners.json** configuration file. + +Check the following table for parameters: + +|Parameter Name |Description |Sample Value | +|---------------------|--------------------------------------------------------------------|:---------------:| +|`jira_project` |The name of the JIRA project where Dow Jones Hammer will create the issue | `AWSSEC` | +|`jira_owner` |The name of the JIRA user to whom Dow Jones Hammer will assign the issue | `Support-Cloud` | +|`jira_parent_ticket` |The JIRA ticket to which Dow Jones Hammer will link the new ticket it creates | `AWSSEC-1234` | +|`slack_owner` |Name(s) of the Slack channels (prefixed by `#`) and/or Slack users that will receive issue reports from Dow Jones Hammer | `["#devops-channel", "bob"]` | + +Sample **ticket_owners.json** section: + +Account-specific settings: +``` +{ + "account": { + "123456789012": { + "jira_project": "", + "jira_owner": "Support-Cloud", + "jira_parent_ticket": "", + "slack_owner": "" + } + }, + "jira_project": "AWSSEC", + "jira_owner": "Support-General", + "jira_parent_ticket": "AWSSEC-1234", + "slack_owner": ["#devops-channel", "bob"] +} +``` + +## 5. Logging + +Dow Jones Hammer uses **CloudWatch Logs** for logging purposes. + +Dow Jones Hammer automatically sets up CloudWatch Log Groups and Log Streams for this issue when you deploy Dow Jones Hammer. + +### 5.1. Issue Identification Logging + +Dow Jones Hammer issue identification functionality uses two Lambda functions: + +* Initialization: this Lambda function selects slave accounts to check for this issue as designated in the Dow Jones Hammer configuration files and triggers the check. +* Identification: this Lambda function identifies this issue for each account/region selected at the previous step. + +You can see the logs for each of these Lambda functions in the following Log Groups: + +|Lambda Function|CloudWatch Log Group Name | +|---------------|--------------------------------------------| +|Initialization |`/aws/lambda/hammer-initiate-elasticsearch-logging`| +|Identification |`/aws/lambda/hammer-describe-elasticsearch-logging`| + +### 5.2. Issue Reporting Logging + +Dow Jones Hammer issue reporting functionality uses ```/aws/ec2/hammer-reporting-remediation``` CloudWatch Log Group for logging. The Log Group contains issue-specific Log Streams named as follows: + +|Designation|CloudWatch Log Stream Name | +|-----------|---------------------------------------------------------| +|Reporting |`reporting.create_elasticsearch_domain_logging_issue_tickets`| + + +### 5.3. Slack Reports + +In case you have enabled Dow Jones Hammer and Slack integration, Dow Jones Hammer sends notifications about issue identification and reporting to the designated Slack channel and/or recipient(s). + +Check [ticket_owners.json](#43-the-ticket_ownersjson-file) configuration for further guidance. + +### 5.4. Using CloudWatch Logs for Dow Jones Hammer + +To access Dow Jones Hammer logs, proceed as follows: + +1. Open **AWS Management Console**. +2. Select **CloudWatch** service. +3. Select **Logs** from the CloudWatch sidebar. +4. Select the log group you want to explore. The log group will open. +5. Select the log stream you want to explore. + +Check [CloudWatch Logs documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) for further guidance. + +## 6. Issue specific details in DynamoDB + +Dow Jones Hammer stores various issue specific details in DynamoDB as a map under `issue_details` key. You can use it to create your own reporting modules. + +|Key |Type |Description |Example | +|-------------|:----:|----------------------------------|------------------------------------------------| +|`name` |string|Elasticsearch domain name |`test-domain` | +|`id` |string|Domain arn |`arn:aws:es:us-east-2:1234567890123:domain/test-domain` | +|`tags` |map |Tags associated with Elasticsearch domain |`{"Name": "TestInstance", "service": "archive"}`| \ No newline at end of file diff --git a/docs/pages/playbook2_insecure_services.md b/docs/pages/playbook2_insecure_services.md index d91e6a76..15a3125c 100644 --- a/docs/pages/playbook2_insecure_services.md +++ b/docs/pages/playbook2_insecure_services.md @@ -89,6 +89,8 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`21`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-security-groups-lambda`| +|`trusted_registrants` |*Optional*. List of registrants of public IPs that should be excluded from the report. |`[]`| Sample **secgrp_unrestricted_access** section of the **config.json** file: @@ -99,7 +101,8 @@ Sample **secgrp_unrestricted_access** section of the **config.json** file: "restricted_ports": [21, 22, 23, 3389, 1433, 1521, 3306, 5432, 27017], "reporting": false, "remediation": false, - "remediation_retention_period": 21 + "remediation_retention_period": 21, + "topic_name": "hammer-describe-security-groups-lambda" } ``` diff --git a/docs/pages/playbook3_inactive_user_keys.md b/docs/pages/playbook3_inactive_user_keys.md index f19dfd56..52d06791 100644 --- a/docs/pages/playbook3_inactive_user_keys.md +++ b/docs/pages/playbook3_inactive_user_keys.md @@ -88,6 +88,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-iam-user-inactive-keys-lambda`| Sample **config.json** section: ``` @@ -97,7 +98,8 @@ Sample **config.json** section: "inactive_criteria_days": "10", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-iam-user-inactive-keys-lambda" } ``` diff --git a/docs/pages/playbook4_keysrotation.md b/docs/pages/playbook4_keysrotation.md index 50a92060..0fa89b16 100644 --- a/docs/pages/playbook4_keysrotation.md +++ b/docs/pages/playbook4_keysrotation.md @@ -84,6 +84,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-iam-user-keys-rotation-lambda`| Sample **config.json** section: @@ -94,7 +95,8 @@ Sample **config.json** section: "rotation_criteria_days": 10, "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-iam-user-keys-rotation-lambda" } ``` diff --git a/docs/pages/playbook5_s3_public_buckets_policy.md b/docs/pages/playbook5_s3_public_buckets_policy.md index 8fcb523d..49f6d663 100644 --- a/docs/pages/playbook5_s3_public_buckets_policy.md +++ b/docs/pages/playbook5_s3_public_buckets_policy.md @@ -92,6 +92,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`7`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-s3-policy-lambda`| Sample **config.json** section: ``` @@ -100,7 +101,8 @@ Sample **config.json** section: "ddb.table_name": "hammer-s3-public-bucket-policy", "reporting": false, "remediation": false - "remediation_retention_period": 7 + "remediation_retention_period": 7, + "topic_name": "hammer-describe-s3-policy-lambda" } ``` diff --git a/docs/pages/playbook6_cloudtrail.md b/docs/pages/playbook6_cloudtrail.md index facc0077..e991c4f8 100644 --- a/docs/pages/playbook6_cloudtrail.md +++ b/docs/pages/playbook6_cloudtrail.md @@ -60,6 +60,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`enabled` |Toggles issue detection for this issue |`true` | |`ddb.table_name` |Name of the DynamoDB table where Dow Jones Hammer will store the identified issues of this type|`hammer-cloudtrails`| |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-cloudtrails-lambda`| Sample **config.json** section: ``` @@ -67,6 +68,7 @@ Sample **config.json** section: "enabled": "true", "ddb.table_name": "hammer-cloudtrails", "reporting": false, + "topic_name": "hammer-describe-cloudtrails-lambda" } ``` diff --git a/docs/pages/playbook7_ebs_unencrypted_volumes.md b/docs/pages/playbook7_ebs_unencrypted_volumes.md index 4fd282c6..1a8e145a 100644 --- a/docs/pages/playbook7_ebs_unencrypted_volumes.md +++ b/docs/pages/playbook7_ebs_unencrypted_volumes.md @@ -64,6 +64,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`ddb.table_name`|The name of the DynamoDB table to which Dow Jones Hammer would record detected issues of this type|`hammer-ebs-volumes-unencrypted` | |`accounts` |*Optional* comma-separated list of accounts to limit check for |`aws.accounts` from `config.json`| |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue |`false` | +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-ebs-unencrypted-volumes-lambda`| Sample **config.json** section: ``` @@ -72,6 +73,7 @@ Sample **config.json** section: "ddb.table_name": "hammer-ebs-volumes-unencrypted", "accounts": ["210987654321"], "reporting": false, + "topic_name": "hammer-describe-ebs-unencrypted-volumes-lambda" } ``` diff --git a/docs/pages/playbook8_ebs_snapshots_public.md b/docs/pages/playbook8_ebs_snapshots_public.md index dec2f0e6..e4327d77 100644 --- a/docs/pages/playbook8_ebs_snapshots_public.md +++ b/docs/pages/playbook8_ebs_snapshots_public.md @@ -87,6 +87,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-ebs-public-snapshots-lambda`| Sample **config.json** section: ``` @@ -95,7 +96,8 @@ Sample **config.json** section: "ddb.table_name": "hammer-ebs-snapshots-public", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-ebs-public-snapshots-lambda" } ``` diff --git a/docs/pages/playbook9_rds_snapshots_public.md b/docs/pages/playbook9_rds_snapshots_public.md index 409383ed..41fcb130 100644 --- a/docs/pages/playbook9_rds_snapshots_public.md +++ b/docs/pages/playbook9_rds_snapshots_public.md @@ -87,6 +87,7 @@ To identify, report, and remediate issues of this type, you should add the follo |`reporting` |Toggle Dow Jones Hammer reporting functionality for this issue type |`false`| |`remediation` |Toggle Dow Jones Hammer automatic remediation functionality for this issue type |`false`| |`remediation_retention_period`|The amount of days to pass between issue detection and its automatic remediation. The value `0` denotes that Dow Jones Hammer will remediate the issue at the next remediation job run.|`0`| +|`topic_name`|Name of the SNS topic to trigger Lambda function from API scan.|`hammer-describe-rds-public-snapshots-lambda`| Sample **config.json** section: ``` @@ -95,7 +96,8 @@ Sample **config.json** section: "ddb.table_name": "hammer-rds-public-snapshots", "reporting": false, "remediation": false, - "remediation_retention_period": 0 + "remediation_retention_period": 0, + "topic_name": "hammer-describe-rds-public-snapshots-lambda" } ``` diff --git a/docs/pages/remediation_backup_rollback.md b/docs/pages/remediation_backup_rollback.md index 8f0f7196..bb1aa4a9 100644 --- a/docs/pages/remediation_backup_rollback.md +++ b/docs/pages/remediation_backup_rollback.md @@ -28,6 +28,15 @@ The following table gives an overview of Dow Jones Hammer remediation functional |[S3 Unencrypted Buckets](playbook11_s3_unencryption.html#3-issue-remediation) | Yes | Yes | |[RDS Unencrypted instances](playbook12_rds_unencryption.html#3-issue-remediation) | `No` | `No` | |[AMIs Public Access](playbook13_amis_public_access.html#3-issue-remediation) | `Yes` | `No` | +|[Redshift Unencryption issues](playbook15__unencryption.html#3-issue-remediation) | `Yes` | `No` | +|[Redshift Public Access issues](playbook16_redshift_public_clusters.html#3-issue-remediation) | `Yes` | `No` | +|[Redshift Logging issues](playbook17_redshift_audit_logging.html#3-issue-remediation) | `No` | `No` | +|[ECS Logging issues](playbook18_ecs_logging.html#3-issue-remediation) | `No` | `No` | +|[ECS Privileged Access issues](playbook19_ecs_privileged_access.html#3-issue-remediation) | `No` | `No` | +|[ECS External Image Source issues](playbook20_ecs_external_image_source.html#3-issue-remediation) | `No` | `No` | +|[Elasticsearch Domains Unencrypted Domains](playbook21_elasticsearch_unencryption.html#3-issue-remediation) | `No` | `No` | +|[Elasticsearch Domains Public Access issues](playbook22_elasticsearch_logging.html#3-issue-remediation) | `Yes` | `Yes` | +|[Elasticsearch Domains Logging issues](playbook23_elasticsearch_logging.html#3-issue-remediation) | `Yes` | `No` | ## 2. How Remediation Backup Works diff --git a/hammer/identification/lambdas/ami-public-access-issues-identification/describe_public_ami_issues.py b/hammer/identification/lambdas/ami-public-access-issues-identification/describe_public_ami_issues.py index 6b957537..fef2f057 100644 --- a/hammer/identification/lambdas/ami-public-access-issues-identification/describe_public_ami_issues.py +++ b/hammer/identification/lambdas/ami-public-access-issues-identification/describe_public_ami_issues.py @@ -58,7 +58,10 @@ def lambda_handler(event, context): issue.issue_details.tags = ami.tags issue.issue_details.name = ami.name issue.issue_details.region = region - if config.publicAMIs.in_whitelist(account_id, ami.id): + + if config.publicAMIs.in_temp_whitelist(account_id, ami.id): + issue.status = IssueStatus.Tempwhitelist + elif config.publicAMIs.in_whitelist(account_id, ami.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py b/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py index b02ea0ec..81f86e3b 100755 --- a/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py +++ b/hammer/identification/lambdas/cloudtrails-issues-identification/describe_cloudtrails.py @@ -56,7 +56,10 @@ def lambda_handler(event, context): issue.issue_details.disabled = checker.disabled issue.issue_details.delivery_errors = checker.delivery_errors issue.add_trails(checker.trails) - if config.cloudtrails.in_whitelist(account_id, region): + + if config.cloudtrails.in_temp_whitelist(account_id, region): + issue.status = IssueStatus.Tempwhitelist + elif config.cloudtrails.in_whitelist(account_id, region): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py b/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py index dee609e9..5a901899 100755 --- a/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py +++ b/hammer/identification/lambdas/ebs-public-snapshots-identification/describe_ebs_public_snapshots.py @@ -57,7 +57,10 @@ def lambda_handler(event, context): issue.issue_details.region = snapshot.account.region issue.issue_details.volume_id = snapshot.volume_id issue.issue_details.tags = snapshot.tags - if config.ebsSnapshot.in_whitelist(account_id, snapshot.id): + + if config.ebsSnapshot.in_temp_whitelist(account_id, snapshot.id): + issue.status = IssueStatus.Tempwhitelist + elif config.ebsSnapshot.in_whitelist(account_id, snapshot.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py b/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py index 6c295aff..b5f3764e 100755 --- a/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py +++ b/hammer/identification/lambdas/ebs-unencrypted-volume-identification/describe_ebs_unencrypted_volumes.py @@ -59,7 +59,10 @@ def lambda_handler(event, context): issue.issue_details.state = volume.state issue.issue_details.attachments = volume.attachments issue.issue_details.tags = volume.tags - if config.ebsVolume.in_whitelist(account_id, volume.id): + + if config.ebsVolume.in_temp_whitelist(account_id, volume.id): + issue.status = IssueStatus.Tempwhitelist + elif config.ebsVolume.in_whitelist(account_id, volume.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/ecs-external-image-source-issues-identification/describe_ecs_external_image_source_issues.py b/hammer/identification/lambdas/ecs-external-image-source-issues-identification/describe_ecs_external_image_source_issues.py new file mode 100644 index 00000000..8694c1e5 --- /dev/null +++ b/hammer/identification/lambdas/ecs-external-image-source-issues-identification/describe_ecs_external_image_source_issues.py @@ -0,0 +1,96 @@ +import json +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.ecs import ECSChecker +from library.aws.utility import Account, DDB +from library.ddb_issues import IssueStatus, ECSExternalImageSourceIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate ECS task definition using external or internal image source. """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.ecs_external_image_source.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking Image source is external or internal for ecs task definitions in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ECSExternalImageSourceIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"ECS task definitions in DDB:\n{open_issues.keys()}") + + checker = ECSChecker(account=account) + if checker.check(): + for task_definition in checker.task_definitions: + logging.debug(f"Checking {task_definition.name}") + if task_definition.external_image: + issue = ECSExternalImageSourceIssue(account_id, task_definition.name) + issue.issue_details.arn = task_definition.arn + issue.issue_details.tags = task_definition.tags + issue.issue_details.container_image_details = task_definition.container_image_details + issue.issue_details.region = task_definition.account.region + + if config.ecs_external_image_source.in_temp_whitelist(account_id, task_definition.name): + issue.status = IssueStatus.Tempwhitelist + elif config.ecs_external_image_source.in_whitelist(account_id, task_definition.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {task_definition.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(task_definition.name, None) + + logging.debug(f"ECS task definitions in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated task definitions + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check ECS task definitions for '{account_id} ({account_name})'") + return + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to identify ECS task definitions external image source checking") + + logging.debug(f"Checked ECS task definitions for '{account_id} ({account_name})'") + + diff --git a/hammer/identification/lambdas/ecs-external-image-source-issues-identification/initiate_to_desc_ecs_external_image_source_issues.py b/hammer/identification/lambdas/ecs-external-image-source-issues-identification/initiate_to_desc_ecs_external_image_source_issues.py new file mode 100644 index 00000000..7cf338ec --- /dev/null +++ b/hammer/identification/lambdas/ecs-external-image-source-issues-identification/initiate_to_desc_ecs_external_image_source_issues.py @@ -0,0 +1,36 @@ +import os +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find ecs task definitions' image source external or internal. """ + set_logging(level=logging.INFO) + logging.debug("Initiating ECS task definitions' image source checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.ecs_external_image_source.enabled: + logging.debug("ECS task definitions' image source checking disabled") + return + + logging.debug("Iterating over each account to initiate ECS task definitions' image source check") + for account_id, account_name in config.ecs_external_image_source.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating ECS task definitions' image source checking for '{account_name}'") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of ECS task definitions' image source checking") + return + + logging.debug("ECS task definitions' image source checking initiation done") diff --git a/hammer/identification/lambdas/ecs-logging-issues-identification/describe_ecs_logging_issues.py b/hammer/identification/lambdas/ecs-logging-issues-identification/describe_ecs_logging_issues.py new file mode 100644 index 00000000..0c595cf4 --- /dev/null +++ b/hammer/identification/lambdas/ecs-logging-issues-identification/describe_ecs_logging_issues.py @@ -0,0 +1,95 @@ +import json +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.ecs import ECSChecker +from library.aws.utility import Account, DDB +from library.ddb_issues import IssueStatus, ECSLoggingIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate ECS logging enabled or not. """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.ecs_logging.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking logging enabled or not for ecs task definitions in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ECSLoggingIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"ECS task definitions in DDB:\n{open_issues.keys()}") + + checker = ECSChecker(account=account) + if checker.check(): + for task_definition in checker.task_definitions: + logging.debug(f"Checking {task_definition.name}") + if not task_definition.is_logging: + issue = ECSLoggingIssue(account_id, task_definition.name) + issue.issue_details.region = task_definition.account.region + issue.issue_details.task_definition_arn = task_definition.arn + issue.issue_details.disabled_logging_container_names = task_definition.disabled_logging_container_names + issue.issue_details.tags = task_definition.tags + + if config.ecs_logging.in_temp_whitelist(account_id, task_definition.name): + issue.status = IssueStatus.Tempwhitelist + elif config.ecs_logging.in_whitelist(account_id, task_definition.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {task_definition.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(task_definition.name, None) + + logging.debug(f"ECS task definitions in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated task definitions + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check ECS task definitions for '{account_id} ({account_name})'") + return + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain audit logging checking") + + logging.debug(f"Checked ECS task definitions for '{account_id} ({account_name})'") + + diff --git a/hammer/identification/lambdas/ecs-logging-issues-identification/initiate_to_desc_ecs_logging_issues.py b/hammer/identification/lambdas/ecs-logging-issues-identification/initiate_to_desc_ecs_logging_issues.py new file mode 100644 index 00000000..99079bdb --- /dev/null +++ b/hammer/identification/lambdas/ecs-logging-issues-identification/initiate_to_desc_ecs_logging_issues.py @@ -0,0 +1,36 @@ +import os +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find clusters logging enabled or not. """ + set_logging(level=logging.INFO) + logging.debug("Initiating ECS Cluster logging checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.ecs_logging.enabled: + logging.debug("ECS cluster logging checking disabled") + return + + logging.debug("Iterating over each account to initiate ECS cluster logging check") + for account_id, account_name in config.ecs_logging.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating ECS cluster logging checking for '{account_name}'") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of ECS cluster logging checking") + return + + logging.debug("ECS clusters logging checking initiation done") diff --git a/hammer/identification/lambdas/ecs-privileged-access-issues-identification/describe_ecs_privileged_access_issues.py b/hammer/identification/lambdas/ecs-privileged-access-issues-identification/describe_ecs_privileged_access_issues.py new file mode 100644 index 00000000..f23edf50 --- /dev/null +++ b/hammer/identification/lambdas/ecs-privileged-access-issues-identification/describe_ecs_privileged_access_issues.py @@ -0,0 +1,94 @@ +import json +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.ecs import ECSChecker +from library.aws.utility import Account, DDB +from library.ddb_issues import IssueStatus, ECSPrivilegedAccessIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate ECS privileged access enabled or not. """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.ecs_privileged_access.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking privileged access enabled or not for ecs task definitions in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ECSPrivilegedAccessIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"ECS privileged access enabled issues in DDB:\n{open_issues.keys()}") + + checker = ECSChecker(account=account) + if checker.check(): + for task_definition in checker.task_definitions: + logging.debug(f"Checking {task_definition.name}") + if task_definition.is_privileged: + issue = ECSPrivilegedAccessIssue(account_id, task_definition.name) + issue.issue_details.arn = task_definition.arn + issue.issue_details.tags = task_definition.tags + issue.issue_details.privileged_container_names = task_definition.privileged_container_names + issue.issue_details.region = task_definition.account.region + if config.ecs_privileged_access.in_temp_whitelist(account_id, task_definition.name): + issue.status = IssueStatus.Tempwhitelist + elif config.ecs_privileged_access.in_whitelist(account_id, task_definition.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {task_definition.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(task_definition.name, None) + + logging.debug(f"ECS privileged access issues in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated task definitions + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check ECS privileged access issues for '{account_id} ({account_name})'") + return + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain privileged access checking") + + logging.debug(f"Checked ECS privileged access issues for '{account_id} ({account_name})'") + + diff --git a/hammer/identification/lambdas/ecs-privileged-access-issues-identification/initiate_to_desc_ecs_privileged_access_issues.py b/hammer/identification/lambdas/ecs-privileged-access-issues-identification/initiate_to_desc_ecs_privileged_access_issues.py new file mode 100644 index 00000000..3555df60 --- /dev/null +++ b/hammer/identification/lambdas/ecs-privileged-access-issues-identification/initiate_to_desc_ecs_privileged_access_issues.py @@ -0,0 +1,36 @@ +import os +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find privileged access enabled or not. """ + set_logging(level=logging.INFO) + logging.debug("Initiating ECS privileged access checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.ecs_privileged_access.enabled: + logging.debug("ECS privileged access checking disabled") + return + + logging.debug("Iterating over each account to initiate ECS privileged access check") + for account_id, account_name in config.ecs_privileged_access.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating ECS privileged access checking for '{account_name}'") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of ECS privileged access checking") + return + + logging.debug("ECS privileged access checking initiation done") diff --git a/hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/describe_elasticsearch_domains_logging_issues.py b/hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/describe_elasticsearch_domains_logging_issues.py new file mode 100644 index 00000000..6eb906ba --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/describe_elasticsearch_domains_logging_issues.py @@ -0,0 +1,92 @@ +import json +import logging + + +from library.logger import set_logging +from library.config import Config +from library.aws.elasticsearch import ESDomainChecker +from library.aws.utility import Account +from library.ddb_issues import IssueStatus, ESLoggingIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import DDB, Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate Elasticsearch domains logging issue """ + set_logging(level=logging.INFO) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.esLogging.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking for Elasticsearch domains logging issue in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ESLoggingIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"Elasticsearch domains logging issue in DDB:\n{open_issues.keys()}") + + checker = ESDomainChecker(account=account) + if checker.check(): + for domain in checker.domains: + if not domain.is_logging: + issue = ESLoggingIssue(account_id, domain.name) + issue.issue_details.region = domain.account.region + issue.issue_details.id = domain.id + issue.issue_details.arn = domain.arn + issue.issue_details.tags = domain.tags + + if config.esLogging.in_temp_whitelist(account_id, domain.name): + issue.status = IssueStatus.Tempwhitelist + elif config.esLogging.in_whitelist(account_id, domain.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {domain.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(domain.name, None) + + logging.debug(f"Elasticsearch domains logging issue in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated Elasticsearch domains + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check Elasticsearch domains logging issue " + f"in '{region}' for '{account_id} ({account_name})'") + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain Elasticsearch domains logging issue checking") + + logging.debug(f"Checked Elasticsearch domains logging issue in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/initiate_to_desc_elasticsearch_domains_logging_issues.py b/hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/initiate_to_desc_elasticsearch_domains_logging_issues.py new file mode 100644 index 00000000..f7ccda13 --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-domain-logging-issues-identification/initiate_to_desc_elasticsearch_domains_logging_issues.py @@ -0,0 +1,36 @@ +import os +import logging + + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find elasticsearch domains logging issue """ + set_logging(level=logging.INFO) + logging.debug("Initiating Elasticsearch domains logging issue checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.esLogging.enabled: + logging.debug("Elasticsearch domains logging issue checking disabled") + return + + logging.debug("Iterating each account to initiate Elasticsearch domains logging issue checking") + for account_id, account_name in config.esLogging.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating Elasticsearch domains logging issue checking for '{account_name}'") + Sns.publish(sns_arn, payload) + except Exception: + logging.exception("Error occurred while initiation of Elasticsearch domains logging issue checking") + return + + logging.debug("Elasticsearch domains logging issue checking initiation done") diff --git a/hammer/identification/lambdas/elasticsearch-public-access-domain-identification/describe_elasticsearch_public_access_domains.py b/hammer/identification/lambdas/elasticsearch-public-access-domain-identification/describe_elasticsearch_public_access_domains.py new file mode 100644 index 00000000..bc20bea8 --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-public-access-domain-identification/describe_elasticsearch_public_access_domains.py @@ -0,0 +1,93 @@ +import json +import logging + + +from library.logger import set_logging +from library.config import Config +from library.aws.elasticsearch import ESDomainChecker +from library.aws.utility import Account +from library.ddb_issues import IssueStatus, ESPublicAccessIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import DDB, Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate Elasticsearch publicly accessible domains """ + set_logging(level=logging.INFO) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.esPublicAccess.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking for Elasticsearch publicly accessible domains in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ESPublicAccessIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"Elasticsearch publicly accessible domains in DDB:\n{open_issues.keys()}") + + checker = ESDomainChecker(account=account) + if checker.check(): + for domain in checker.domains: + if domain.public: + issue = ESPublicAccessIssue(account_id, domain.name) + issue.issue_details.region = domain.account.region + issue.issue_details.id = domain.id + issue.issue_details.arn = domain.arn + issue.issue_details.tags = domain.tags + issue.issue_details.policy = domain.policy + + if config.esPublicAccess.in_temp_whitelist(account_id, domain.name): + issue.status = IssueStatus.Tempwhitelist + elif config.esPublicAccess.in_whitelist(account_id, domain.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {domain.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(domain.name, None) + + logging.debug(f"Elasticsearch publicly accessible domains in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated Elasticsearch domains + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check Elasticsearch publicly accessible domains " + f"in '{region}' for '{account_id} ({account_name})'") + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain Elasticsearch publicly accessible domains checking") + + logging.debug(f"Checked Elasticsearch publicly accessible domains in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/elasticsearch-public-access-domain-identification/initiate_to_desc_elasticsearch_public_access_domains.py b/hammer/identification/lambdas/elasticsearch-public-access-domain-identification/initiate_to_desc_elasticsearch_public_access_domains.py new file mode 100644 index 00000000..f097188e --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-public-access-domain-identification/initiate_to_desc_elasticsearch_public_access_domains.py @@ -0,0 +1,36 @@ +import os +import logging + + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find publicly accessible elasticsearch domains """ + set_logging(level=logging.INFO) + logging.debug("Initiating publicly accessible Elasticsearch domains checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.esPublicAccess.enabled: + logging.debug("Elasticsearch publicly accessible domains checking disabled") + return + + logging.debug("Iterating each account to initiate Elasticsearch publicly accessible domains checking") + for account_id, account_name in config.esPublicAccess.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating Elasticsearch publicly accessible domains checking for '{account_name}'") + Sns.publish(sns_arn, payload) + except Exception: + logging.exception("Error occurred while initiation of Elasticsearch publicly accessible domains checking") + return + + logging.debug("Elasticsearch publicly accessible domains checking initiation done") diff --git a/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/describe_elasticsearch_unencrypted_domains.py b/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/describe_elasticsearch_unencrypted_domains.py new file mode 100644 index 00000000..b039d851 --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/describe_elasticsearch_unencrypted_domains.py @@ -0,0 +1,94 @@ +import json +import logging + + +from library.logger import set_logging +from library.config import Config +from library.aws.elasticsearch import ESDomainChecker +from library.aws.utility import Account +from library.ddb_issues import IssueStatus, ESEncryptionIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import DDB, Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate Elasticsearch unencrypted domains """ + set_logging(level=logging.INFO) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.esEncrypt.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking for Elasticsearch unencrypted domains in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ESEncryptionIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"Elasticsearch unencrypted domains in DDB:\n{open_issues.keys()}") + + checker = ESDomainChecker(account=account) + if checker.check(): + for domain in checker.domains: + if not (domain.encrypted_at_rest and domain.encrypted_at_transit): + issue = ESEncryptionIssue(account_id, domain.name) + issue.issue_details.region = domain.account.region + issue.issue_details.id = domain.id + issue.issue_details.arn = domain.arn + issue.issue_details.tags = domain.tags + issue.issue_details.encrypted_at_rest = domain.encrypted_at_rest + issue.issue_details.encrypted_at_transit = domain.encrypted_at_transit + + if config.esEncrypt.in_temp_whitelist(account_id, domain.name): + issue.status = IssueStatus.Tempwhitelist + elif config.esEncrypt.in_whitelist(account_id, domain.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {domain.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(domain.name, None) + + logging.debug(f"Elasticsearch unencrypted domains in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated Elasticsearch domains + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check Elasticsearch unencrypted domains " + f"in '{region}' for '{account_id} ({account_name})'") + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain Elasticsearch unencrypted domains checking") + + logging.debug(f"Checked Elasticsearch unencrypted domains in '{region}' for '{account_id} ({account_name})'") diff --git a/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/initiate_to_desc_elasticsearch_unencrypted_domains.py b/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/initiate_to_desc_elasticsearch_unencrypted_domains.py new file mode 100644 index 00000000..500687e2 --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/initiate_to_desc_elasticsearch_unencrypted_domains.py @@ -0,0 +1,36 @@ +import os +import logging + + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find unencrypted elasticsearch domains """ + set_logging(level=logging.INFO) + logging.debug("Initiating unencrypted Elasticsearch domains checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.esEncrypt.enabled: + logging.debug("Elasticsearch unencrypted domains checking disabled") + return + + logging.debug("Iterating each account to initiate Elasticsearch unencrypted domains checking") + for account_id, account_name in config.esEncrypt.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating Elasticsearch unencrypted domains checking for '{account_name}'") + Sns.publish(sns_arn, payload) + except Exception: + logging.exception("Error occurred while initiation of Elasticsearch unencrypted domains checking") + return + + logging.debug("Elasticsearch unencrypted domains checking initiation done") diff --git a/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/requirements.txt b/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/requirements.txt new file mode 100644 index 00000000..7001ed12 --- /dev/null +++ b/hammer/identification/lambdas/elasticsearch-unencrypted-domain-identification/requirements.txt @@ -0,0 +1,2 @@ +boto3==1.9.42 +requests \ No newline at end of file diff --git a/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py b/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py index b85e0bc2..2727f22d 100755 --- a/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py +++ b/hammer/identification/lambdas/iam-keyrotation-issues-identification/describe_iam_key_rotation.py @@ -56,7 +56,12 @@ def lambda_handler(event, context): issue = IAMKeyRotationIssue(account_id, key.id) issue.issue_details.username = user.id issue.issue_details.create_date = key.create_date.isoformat() - if config.iamUserKeysRotation.in_whitelist(account_id, key.id) or config.iamUserKeysRotation.in_whitelist(account_id, user.id): + + if config.iamUserKeysRotation.in_temp_whitelist(account_id, key.id) \ + or config.iamUserKeysRotation.in_temp_whitelist(account_id, user.id): + issue.status = IssueStatus.Tempwhitelist + elif config.iamUserKeysRotation.in_whitelist(account_id, key.id) \ + or config.iamUserKeysRotation.in_whitelist(account_id, user.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py b/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py index c1db9fac..9d5f6e9f 100755 --- a/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py +++ b/hammer/identification/lambdas/iam-user-inactive-keys-identification/describe_iam_accesskey_details.py @@ -57,7 +57,12 @@ def lambda_handler(event, context): issue.issue_details.username = user.id issue.issue_details.last_used = key.last_used.isoformat() issue.issue_details.create_date = key.create_date.isoformat() - if config.iamUserInactiveKeys.in_whitelist(account_id, key.id) or config.iamUserInactiveKeys.in_whitelist(account_id, user.id): + + if config.iamUserInactiveKeys.in_temp_whitelist(account_id, key.id) \ + or config.iamUserInactiveKeys.in_temp_whitelist(account_id, user.id): + issue.status = IssueStatus.Tempwhitelist + elif config.iamUserInactiveKeys.in_whitelist(account_id, key.id) \ + or config.iamUserInactiveKeys.in_whitelist(account_id, user.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py b/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py index 6d155389..c0f0bd07 100755 --- a/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py +++ b/hammer/identification/lambdas/rds-public-snapshots-identification/describe_rds_public_snapshots.py @@ -59,7 +59,10 @@ def lambda_handler(event, context): issue.issue_details.region = snapshot.account.region issue.issue_details.engine = snapshot.engine issue.issue_details.tags = snapshot.tags - if config.rdsSnapshot.in_whitelist(account_id, snapshot.id): + + if config.rdsSnapshot.in_temp_whitelist(account_id, snapshot.id): + issue.status = IssueStatus.Tempwhitelist + elif config.rdsSnapshot.in_whitelist(account_id, snapshot.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py b/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py index bc84e972..34fa4d32 100644 --- a/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py +++ b/hammer/identification/lambdas/rds-unencrypted-instance-identification/describe_rds_instance_encryption.py @@ -59,7 +59,10 @@ def lambda_handler(event, context): issue.issue_details.region = instance.account.region issue.issue_details.engine = instance.engine issue.issue_details.tags = instance.tags - if config.rdsEncrypt.in_whitelist(account_id, instance.id): + + if config.rdsEncrypt.in_temp_whitelist(account_id, instance.id): + issue.status = IssueStatus.Tempwhitelist + elif config.rdsEncrypt.in_whitelist(account_id, instance.id): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/redshift-audit-logging-issues-identification/describe_redshift_logging_issues.py b/hammer/identification/lambdas/redshift-audit-logging-issues-identification/describe_redshift_logging_issues.py new file mode 100644 index 00000000..06e35ef7 --- /dev/null +++ b/hammer/identification/lambdas/redshift-audit-logging-issues-identification/describe_redshift_logging_issues.py @@ -0,0 +1,93 @@ +import json +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account, DDB +from library.ddb_issues import IssueStatus, RedshiftLoggingIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate Redshift logging enabled or not. """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.redshift_logging.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking logging enabled or not for Redshift clusters in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, RedshiftLoggingIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"Redshift clusters in DDB:\n{open_issues.keys()}") + + checker = RedshiftClusterChecker(account=account) + if checker.check(): + for cluster in checker.clusters: + logging.debug(f"Checking {cluster.name}") + if not cluster.is_logging: + issue = RedshiftLoggingIssue(account_id, cluster.name) + issue.issue_details.tags = cluster.tags + issue.issue_details.region = cluster.account.region + + if config.redshift_logging.in_temp_whitelist(account_id, cluster.name): + issue.status = IssueStatus.Tempwhitelist + elif config.redshift_logging.in_whitelist(account_id, cluster.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {cluster.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(cluster.name, None) + + logging.debug(f"Redshift Clusters in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated clusters + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check Redshift clusters for '{account_id} ({account_name})'") + return + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain audit logging checking") + + logging.debug(f"Checked Redshift Clusters for '{account_id} ({account_name})'") + + diff --git a/hammer/identification/lambdas/redshift-audit-logging-issues-identification/initiate_to_desc_redshift_logging_issues.py b/hammer/identification/lambdas/redshift-audit-logging-issues-identification/initiate_to_desc_redshift_logging_issues.py new file mode 100644 index 00000000..9efefa04 --- /dev/null +++ b/hammer/identification/lambdas/redshift-audit-logging-issues-identification/initiate_to_desc_redshift_logging_issues.py @@ -0,0 +1,36 @@ +import os +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find clusters logging enabled or not. """ + set_logging(level=logging.INFO) + logging.debug("Initiating Redshift Cluster logging checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.redshift_logging.enabled: + logging.debug("Redshift cluster logging checking disabled") + return + + logging.debug("Iterating over each account to initiate Redshift cluster logging check") + for account_id, account_name in config.redshift_logging.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating Redshift cluster logging checking for '{account_name}'") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of Redshift cluster logging checking") + return + + logging.debug("Redshift clusters logging checking initiation done") diff --git a/hammer/identification/lambdas/redshift-cluster-public-access-identification/describe_redshift_cluster_public_access.py b/hammer/identification/lambdas/redshift-cluster-public-access-identification/describe_redshift_cluster_public_access.py new file mode 100644 index 00000000..7db33692 --- /dev/null +++ b/hammer/identification/lambdas/redshift-cluster-public-access-identification/describe_redshift_cluster_public_access.py @@ -0,0 +1,93 @@ +import json +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account, DDB +from library.ddb_issues import IssueStatus, RedshiftPublicAccessIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate Redshift cluster publicly accessible or not. """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.redshift_public_access.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking for publicly accessible Redshift clusters in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, RedshiftPublicAccessIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"Redshift clusters in DDB:\n{open_issues.keys()}") + + checker = RedshiftClusterChecker(account=account) + if checker.check(): + for cluster in checker.clusters: + logging.debug(f"Checking {cluster.name}") + if cluster.is_public: + issue = RedshiftPublicAccessIssue(account_id, cluster.name) + issue.issue_details.tags = cluster.tags + issue.issue_details.region = cluster.account.region + + if config.redshift_public_access.in_temp_whitelist(account_id, cluster.name): + issue.status = IssueStatus.Tempwhitelist + elif config.redshift_public_access.in_whitelist(account_id, cluster.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {cluster.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(cluster.name, None) + + logging.debug(f"Redshift Clusters in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated clusters + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check Redshift clusters for '{account_id} ({account_name})'") + return + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain public access issues checking") + + logging.debug(f"Checked Redshift Clusters for '{account_id} ({account_name})'") + + diff --git a/hammer/identification/lambdas/redshift-cluster-public-access-identification/initiate_to_desc_redshift_cluster_public_access.py b/hammer/identification/lambdas/redshift-cluster-public-access-identification/initiate_to_desc_redshift_cluster_public_access.py new file mode 100644 index 00000000..2fc69218 --- /dev/null +++ b/hammer/identification/lambdas/redshift-cluster-public-access-identification/initiate_to_desc_redshift_cluster_public_access.py @@ -0,0 +1,36 @@ +import os +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find clusters have public access or not. """ + set_logging(level=logging.INFO) + logging.debug("Initiating Redshift Cluster public access checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.redshift_public_access.enabled: + logging.debug("Redshift cluster public access checking disabled") + return + + logging.debug("Iterating over each account to initiate Redshift cluster public access check") + for account_id, account_name in config.redshift_public_access.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating Redshift cluster public access checking for '{account_name}'") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of Redshift cluster public access checking") + return + + logging.debug("Redshift clusters public access checking initiation done") diff --git a/hammer/identification/lambdas/redshift-unencrypted-cluster-identification/describe_redshift_encryption.py b/hammer/identification/lambdas/redshift-unencrypted-cluster-identification/describe_redshift_encryption.py new file mode 100644 index 00000000..4e1c5de3 --- /dev/null +++ b/hammer/identification/lambdas/redshift-unencrypted-cluster-identification/describe_redshift_encryption.py @@ -0,0 +1,92 @@ +import json +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account, DDB +from library.ddb_issues import IssueStatus, RedshiftEncryptionIssue +from library.ddb_issues import Operations as IssueOperations +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to evaluate Redshift cluster encryption """ + set_logging(level=logging.DEBUG) + + try: + payload = json.loads(event["Records"][0]["Sns"]["Message"]) + account_id = payload['account_id'] + account_name = payload['account_name'] + # get the last region from the list to process + region = payload['regions'].pop() + # if request_id is present in payload then this lambda was called from the API + request_id = payload.get('request_id', None) + except Exception: + logging.exception(f"Failed to parse event\n{event}") + return + + try: + config = Config() + + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(config.redshiftEncrypt.ddb_table_name) + + account = Account(id=account_id, + name=account_name, + region=region, + role_name=config.aws.role_name_identification) + if account.session is None: + return + + logging.debug(f"Checking for unencrypted Redshift clusters policies in {account}") + + # existing open issues for account to check if resolved + open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, RedshiftEncryptionIssue) + # make dictionary for fast search by id + # and filter by current region + open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region} + logging.debug(f"Redshift clusters in DDB:\n{open_issues.keys()}") + + checker = RedshiftClusterChecker(account=account) + if checker.check(): + for cluster in checker.clusters: + logging.debug(f"Checking {cluster.name}") + if not cluster.is_encrypt: + issue = RedshiftEncryptionIssue(account_id, cluster.name) + issue.issue_details.tags = cluster.tags + issue.issue_details.region = cluster.account.region + + if config.redshiftEncrypt.in_temp_whitelist(account_id, cluster.name): + issue.status = IssueStatus.Tempwhitelist + elif config.redshiftEncrypt.in_whitelist(account_id, cluster.name): + issue.status = IssueStatus.Whitelisted + else: + issue.status = IssueStatus.Open + logging.debug(f"Setting {cluster.name} status {issue.status}") + IssueOperations.update(ddb_table, issue) + # remove issue id from issues_list_from_db (if exists) + # as we already checked it + open_issues.pop(cluster.name, None) + + logging.debug(f"Redshift Clusters in DDB:\n{open_issues.keys()}") + # all other unresolved issues in DDB are for removed/remediated clusters + for issue in open_issues.values(): + IssueOperations.set_status_resolved(ddb_table, issue) + + # track the progress of API request to scan specific account/region/feature + if request_id: + api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name) + DDB.track_progress(api_table, request_id) + except Exception: + logging.exception(f"Failed to check Redshift clusters for '{account_id} ({account_name})'") + return + + # push SNS messages until the list with regions to check is empty + if len(payload['regions']) > 0: + try: + Sns.publish(payload["sns_arn"], payload) + except Exception: + logging.exception("Failed to chain insecure services checking") + + logging.debug(f"Checked Redshift Clusters for '{account_id} ({account_name})'") \ No newline at end of file diff --git a/hammer/identification/lambdas/redshift-unencrypted-cluster-identification/initiate_to_desc_redshift_encryption.py b/hammer/identification/lambdas/redshift-unencrypted-cluster-identification/initiate_to_desc_redshift_encryption.py new file mode 100644 index 00000000..cc499bf0 --- /dev/null +++ b/hammer/identification/lambdas/redshift-unencrypted-cluster-identification/initiate_to_desc_redshift_encryption.py @@ -0,0 +1,36 @@ +import os +import logging + +from library.logger import set_logging +from library.config import Config +from library.aws.utility import Sns + + +def lambda_handler(event, context): + """ Lambda handler to initiate to find unencrypted Redshift clusters """ + set_logging(level=logging.INFO) + logging.debug("Initiating Redshift Clusters checking") + + try: + sns_arn = os.environ["SNS_ARN"] + config = Config() + + if not config.redshiftEncrypt.enabled: + logging.debug("Redshift clusters checking disabled") + return + + logging.debug("Iterating over each account to initiate Redshift Clusters check") + for account_id, account_name in config.redshiftEncrypt.accounts.items(): + payload = {"account_id": account_id, + "account_name": account_name, + "regions": config.aws.regions, + "sns_arn": sns_arn + } + logging.debug(f"Initiating unencrypted Redshift clusters checking for '{account_name}'") + Sns.publish(sns_arn, payload) + + except Exception: + logging.exception("Error occurred while initiation of unencrypted Redshift cluster checking") + return + + logging.debug("unencrypted Redshift clusters checking initiation done") diff --git a/hammer/identification/lambdas/requirements.txt b/hammer/identification/lambdas/requirements.txt index 663bd1f6..59e7f8f4 100755 --- a/hammer/identification/lambdas/requirements.txt +++ b/hammer/identification/lambdas/requirements.txt @@ -1 +1,2 @@ -requests \ No newline at end of file +requests +ipwhois \ No newline at end of file diff --git a/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py b/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py index 6f8f20fa..30b69fe3 100755 --- a/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py +++ b/hammer/identification/lambdas/s3-acl-issues-identification/describe_s3_bucket_acl.py @@ -55,7 +55,10 @@ def lambda_handler(event, context): issue.issue_details.owner = bucket.owner issue.issue_details.public_acls = bucket.get_public_acls() issue.issue_details.tags = bucket.tags - if config.s3acl.in_whitelist(account_id, bucket.name): + + if config.s3acl.in_temp_whitelist(account_id, bucket.name): + issue.status = IssueStatus.Tempwhitelist + elif config.s3acl.in_whitelist(account_id, bucket.name): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py b/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py index 2ac13ae0..e852cf4b 100755 --- a/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py +++ b/hammer/identification/lambdas/s3-policy-issues-identification/describe_s3_bucket_policy.py @@ -55,7 +55,10 @@ def lambda_handler(event, context): issue.issue_details.owner = bucket.owner issue.issue_details.tags = bucket.tags issue.issue_details.policy = bucket.policy - if config.s3policy.in_whitelist(account_id, bucket.name): + + if config.s3policy.in_temp_whitelist(account_id, bucket.name): + issue.status = IssueStatus.Tempwhitelist + elif config.s3policy.in_whitelist(account_id, bucket.name): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py b/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py index ecf8e766..4e14c5fb 100644 --- a/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py +++ b/hammer/identification/lambdas/s3-unencrypted-bucket-issues-identification/describe_s3_encryption.py @@ -54,7 +54,10 @@ def lambda_handler(event, context): issue = S3EncryptionIssue(account_id, bucket.name) issue.issue_details.owner = bucket.owner issue.issue_details.tags = bucket.tags - if config.s3Encrypt.in_whitelist(account_id, bucket.name): + + if config.s3Encrypt.in_temp_whitelist(account_id, bucket.name): + issue.status = IssueStatus.Tempwhitelist + elif config.s3Encrypt.in_whitelist(account_id, bucket.name): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py b/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py index 5228a266..74a2a47f 100755 --- a/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py +++ b/hammer/identification/lambdas/sg-issues-identification/describe_sec_grps_unrestricted_access.py @@ -68,7 +68,11 @@ def lambda_handler(event, context): for ip_range in perm.ip_ranges: if not ip_range.restricted: issue.add_perm(perm.protocol, perm.from_port, perm.to_port, ip_range.cidr, ip_range.status) - if config.sg.in_whitelist(account_id, f"{sg.vpc_id}:{sg.name}") or \ + + if config.sg.in_temp_whitelist(account_id, f"{sg.vpc_id}:{sg.name}")or \ + config.sg.in_temp_whitelist(account_id, sg.id): + issue.status = IssueStatus.Tempwhitelist + elif config.sg.in_whitelist(account_id, f"{sg.vpc_id}:{sg.name}") or \ config.sg.in_whitelist(account_id, sg.id): issue.status = IssueStatus.Whitelisted else: diff --git a/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py b/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py index 63a02b12..0750b84c 100644 --- a/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py +++ b/hammer/identification/lambdas/sqs-public-policy-identification/describe_sqs_public_policy.py @@ -59,7 +59,10 @@ def lambda_handler(event, context): issue.issue_details.name = queue.name issue.issue_details.region = queue.account.region issue.issue_details.policy = queue.policy - if config.sqspolicy.in_whitelist(account_id, queue.url): + + if config.sqspolicy.in_temp_whitelist(account_id, queue.url): + issue.status = IssueStatus.Tempwhitelist + elif config.sqspolicy.in_whitelist(account_id, queue.url): issue.status = IssueStatus.Whitelisted else: issue.status = IssueStatus.Open diff --git a/hammer/library/aws/ebs.py b/hammer/library/aws/ebs.py index 1a9c66d8..c33b9c7c 100755 --- a/hammer/library/aws/ebs.py +++ b/hammer/library/aws/ebs.py @@ -106,8 +106,11 @@ def check(self, ids=None, tags=None): {'Name': f"tag:{key}", 'Values': value if isinstance(value, list) else [value]}, ) + volume_details = [] try: - volume_details = self.account.client("ec2").describe_volumes(**args)["Volumes"] + paginator = self.account.client("ec2").get_paginator('describe_volumes') + for page in paginator.paginate(**args): + volume_details.extend(page["Volumes"]) except ClientError as err: if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: logging.error(f"Access denied in {self.account} " diff --git a/hammer/library/aws/ec2.py b/hammer/library/aws/ec2.py index 76c3775d..af02f2d5 100755 --- a/hammer/library/aws/ec2.py +++ b/hammer/library/aws/ec2.py @@ -285,7 +285,7 @@ def __str__(self): f")") def modify_image_attribute(self): - EC2Operations.modify_image_attribute(self.account.client("ec2"), self.id) + return EC2Operations.modify_image_attribute(self.account.client("ec2"), self.id) class PublicAMIChecker(object): diff --git a/hammer/library/aws/ecs.py b/hammer/library/aws/ecs.py new file mode 100644 index 00000000..f22e3a53 --- /dev/null +++ b/hammer/library/aws/ecs.py @@ -0,0 +1,205 @@ +import logging + +from botocore.exceptions import ClientError +from library.utility import timeit +from collections import namedtuple +from library.aws.utility import convert_tags +from library.config import Config + +# structure which describes EC2 instance +ECSCluster_Details = namedtuple('ECSCluster_Details', [ + # cluster_id + 'cluster_arn', + # subnet_group_id + 'cluster_instance_arn' +]) + + +class ECSClusterOperations(object): + @classmethod + @timeit + def get_ecs_instance_security_groups(cls, ec2_client, ecs_client, group_id): + """ Retrieve ecs clusters meta data with security group attached + + :param ec2_client: boto3 ec2 client + :param ecs_client: boto3 ECS client + :param group_id: security group id + + :return: list with ecs clusters details + """ + # describe ecs instances with security group attached + ecs_instances = [] + + # this will include Clusters + clusters_res = ecs_client.list_clusters() + for cluster_arn in clusters_res["clusterArns"]: + list_container_instances = ecs_client.list_container_instances( + cluster=cluster_arn + ) + + for instance_arn in list_container_instances["containerInstanceArns"]: + container_instance = ecs_client.describe_container_instances( + cluster=cluster_arn, + containerInstances=[ + instance_arn, + ] + ) + ec2_instance_id = container_instance["containerInstances"][0]["ec2InstanceId"] + ec2_instance = ec2_client.describe_instances(InstanceIds=[ec2_instance_id])['Reservations'][0]["Instances"][0] + if group_id in str(ec2_instance["SecurityGroups"]): + ecs_instances.append(ECSCluster_Details( + cluster_arn=cluster_arn, + cluster_instance_arn=instance_arn + )) + + return ecs_instances + + +class ECSTaskDefinitions(object): + """ + Basic class for ECS task definitions. + + """ + + def __init__(self, account, name, arn, tags, is_logging=None, disabled_logging_container_names=None, + is_privileged=None, privileged_container_names=None, external_image=None, + container_image_details=None): + """ + + :param account: `Account` instance where ECS task definition is present + :param name: name of the task definition + :param arn: arn of the task definition + :param tags: tags of task definition. + :param is_logging: boolean. Task definition's container logging is enabled or not + :param disabled_logging_container_names: List of containers which logging disabled. + :param is_privileged: boolean + :param privileged_container_names: List of containers which privileged access enabled + :param external_image: boolean + :param container_image_details: List of containers which image source is taken from external + """ + + self.account = account + self.name = name + self.arn = arn + self.tags = convert_tags(tags) + self.is_logging = is_logging + self.disabled_logging_container_names = disabled_logging_container_names + self.is_privileged = is_privileged + self.privileged_container_names = privileged_container_names + self.external_image = external_image + self.container_image_details = container_image_details + + +class ECSChecker(object): + """ + Basic class for checking ecs task definition's logging/privileged access/image source in account/region. + Encapsulates check settings and discovered task definition's containers. + """ + + def __init__(self, account): + """ + :param account: `Account` task definitions to check + + """ + self.account = account + self.task_definitions = [] + + def validate_image_source(self, image): + config = Config() + + is_external = True + safe_image_sources = config.ecs_external_image_source.safe_image_sources + for image_source in safe_image_sources: + if image_source in image: + is_external = False + break + + return is_external + + def check(self, task_definitions=None): + """ + Walk through clusters in the account/region and check them. + Put all ECS task definition's container details. + + :return: boolean. True - if check was successful, + False - otherwise + """ + try: + # AWS does not support filtering, so get all task definition family details for account + response = self.account.client("ecs").list_task_definition_families() + except ClientError as err: + if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: + logging.error(f"Access denied in {self.account} " + f"(ecs:{err.operation_name})") + else: + logging.exception(f"Failed to list task definitions in {self.account}") + return False + + if "families" in response: + for task_definition_name in response["families"]: + if task_definitions is not None and task_definition_name not in task_definitions: + continue + + tags = {} + container_image_details = [] + disabled_logging_container_names = [] + privileged_container_names = [] + external_image = False + try: + task_definition = self.account.client("ecs").describe_task_definition( + taskDefinition=task_definition_name + )['taskDefinition'] + task_definition_arn = task_definition["taskDefinitionArn"] + if "containerDefinitions" in task_definition: + for container_definition in task_definition['containerDefinitions']: + container_name = container_definition["name"] + if container_definition.get('logConfiguration') is None: + disabled_logging_container_names.append(container_name) + + if container_definition.get('privileged') is not None \ + and container_definition['privileged']: + privileged_container_names.append(container_name) + + image = container_definition.get('image') + if image is not None: + external_image = self.validate_image_source(image) + image_details = {} + if external_image: + image_details["container_name"] = container_name + image_details["image_url"] = image + container_image_details.append(image_details) + + if len(disabled_logging_container_names) > 0: + logging_enabled = False + else: + logging_enabled = True + + if len(privileged_container_names) > 0: + is_privileged = True + else: + is_privileged = False + + if "Tags" in task_definition: + tags = task_definition["Tags"] + task_definition_details = ECSTaskDefinitions(account=self.account, + name=task_definition_name, + arn=task_definition_arn, + tags=tags, + is_logging=logging_enabled, + disabled_logging_container_names=disabled_logging_container_names, + is_privileged=is_privileged, + privileged_container_names=privileged_container_names, + external_image=external_image, + container_image_details=container_image_details, + ) + self.task_definitions.append(task_definition_details) + except ClientError as err: + if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: + logging.error(f"Access denied in {self.account} " + f"(ecs:{err.operation_name})") + else: + logging.error(f"Failed to describe task definitions in {self.account} " + f"for task {task_definition_name}") + continue + + return True diff --git a/hammer/library/aws/elasticsearch.py b/hammer/library/aws/elasticsearch.py new file mode 100644 index 00000000..577eafe7 --- /dev/null +++ b/hammer/library/aws/elasticsearch.py @@ -0,0 +1,360 @@ +import json +import logging +import pathlib + +from datetime import datetime, timezone +from botocore.exceptions import ClientError +from collections import namedtuple +from library.utility import timeit +from library.utility import jsonDumps +from library.aws.utility import convert_tags +from library.aws.s3 import S3Operations + +# structure which describes Elastic search domains +ElasticSearchDomain_Details = namedtuple('ElasticSearchDomain', [ + # domain name + 'domain_name', + # domain arn + 'domain_arn', + # vpc_id + 'vpc_id' +]) + + +class ElasticSearchOperations: + @classmethod + @timeit + def get_elasticsearch_details_of_sg_associated(cls, elasticsearch_client, group_id): + """ Retrieve elastic search details meta data with security group attached + + :param elasticsearch_client: boto3 elastic search client + :param group_id: security group id + + :return: list with elastic search details + """ + # describe elastic search domain details with security group attached. + domains_list = [] + + elasticsearch_response = elasticsearch_client.list_domain_names() + for domain in elasticsearch_response["DomainNames"]: + domain_name = domain["DomainName"] + domain_details = elasticsearch_client.describe_elasticsearch_domain( + DomainName=domain_name + )["DomainStatus"] + if group_id in str(domain_details): + domains_list.append(ElasticSearchDomain_Details( + domain_name=domain_name, + domain_arn=domain_details["ARN"], + vpc_id=domain_details["VPCOptions"]["VPCId"] + )) + + return domains_list + + @staticmethod + def put_domain_policy(es_client, domain_name, policy): + """ + Replaces a policy on a domain. If the domain already has a policy, one in this request completely replaces it. + + :param es_client: Elasticsearch boto3 client + :param domain_name: Elasticsearch domain where to update policy on + :param policy: `dict` or `str` with policy. `Dict` will be transformed to string using pretty json.dumps(). + + :return: nothing + """ + policy_json = jsonDumps(policy) if isinstance(policy, dict) else policy + es_client.update_elasticsearch_domain_config( + DomainName=domain_name, + AccessPolicies=policy_json, + ) + + @staticmethod + def retrieve_loggroup_arn(cw_client, domain_log_group_name): + """ + This method used to retrieve cloud watch log group arn details if log group is available. If not, create a + cloudwatch log group and returns arn of newly created log group + + :param cw_client: cloudwatch logs boto3 client + :param domain_log_group_name: Elasticsearch domain's log group name + :return: + """ + log_groups = cw_client.describe_log_groups() + + log_group_arn = None + for log_group in log_groups["logGroups"]: + log_group_name = log_group["logGroupName"] + if log_group_name == domain_log_group_name: + log_group_arn = log_group["arn"] + + if log_group_arn: + """ + In order to successfully deliver the logs to your CloudWatch Logs log group, + Amazon Elasticsearch Service (AES) will need access to two CloudWatch Logs API calls: + 1. CreateLogStream: Create a CloudWatch Logs log stream for the log group you specified + 2. PutLogEvents: Deliver CloudTrail events to the CloudWatch Logs log stream + + Adding resource policy that grants above access. + """ + policy_name = "AES-" + domain_log_group_name + "-Application-logs" + policy_doc = {} + statement = {} + principal = {} + action = [] + principal["Service"] = "es.amazonaws.com" + action.append("logs:PutLogEvents") + action.append("logs:CreateLogStream") + statement["Effect"] = "Allow" + statement["Principal"] = principal + statement["Action"] = action + statement["Resource"] = log_group_arn + + policy_doc["Statement"] = statement + + cw_client.put_resource_policy( + policyName=policy_name, + policyDocument=str(json.dumps(policy_doc)) + ) + return log_group_arn + + @staticmethod + def set_domain_logging(es_client, cw_client, domain_name): + """ + + :param es_client: elastic search boto3 client + :param cw_client: cloudwatch logs boto3 client + :param domain_name: elastic search domain name + :return: + """ + domain_log_group_name = "/aws/aes/domains/" + domain_name + "/application-logs" + log_group_arn = ElasticSearchOperations.retrieve_loggroup_arn(cw_client, domain_log_group_name) + if not log_group_arn: + cw_client.create_log_group(logGroupName=domain_log_group_name) + log_group_arn = ElasticSearchOperations.retrieve_loggroup_arn(cw_client, domain_log_group_name) + + es_client.update_elasticsearch_domain_config( + DomainName=domain_name, + LogPublishingOptions={ + 'ES_APPLICATION_LOGS': + { + 'CloudWatchLogsLogGroupArn': log_group_arn, + 'Enabled': True + } + } + ) + + @classmethod + def validate_access_policy(cls, policy_details): + """ + + :param policy_details: + :return: + """ + public_policy = False + for statement in policy_details.get("Statement", []): + public_policy = S3Operations.public_statement(statement) + + return public_policy + + +class ESDomainDetails(object): + """ + Basic class for ElasticSearch domain details. + + """ + + def __init__(self, account, name, id, arn, tags=None, is_logging=None, encrypted_at_rest=None, + encrypted_at_transit=None, policy=None): + """ + + :param account: `Account` instance where Elasticsearch domain is present + :param name: name of the Elasticsearch domain + :param id: Elasticsearch domain id. + :param arn: arn of the Elasticsearch domain + :param tags: tags of Elasticsearch domain. + :param is_logging: flag for logging enabled or not. + :param encrypted_at_rest: flag for encryption enabled at rest or not + :param encrypted_at_transit: flag for encryption enabled at transit or not + :param policy: + """ + + self.account = account + self.name = name + self.id = id + self.arn = arn + self.is_logging = is_logging + self.encrypted_at_rest = encrypted_at_rest + self.encrypted_at_transit = encrypted_at_transit + self._policy = json.loads(policy) if policy else {} + self.backup_filename = pathlib.Path(f"{self.name}.json") + self.tags = convert_tags(tags) + + @property + def policy(self): + """ + :return: pretty formatted string with S3 bucket policy + """ + return jsonDumps(self._policy) + + @property + def public(self): + """ + :return: boolean, True - if Elasticsearch domain policy allows public access + False - otherwise + """ + return ElasticSearchOperations.validate_access_policy(self._policy) + + def backup_policy_s3(self, s3_client, bucket): + """ + Backup Elasticsearch policy json to S3. + + :param s3_client: S3 boto3 client + :param bucket: S3 bucket name where to put backup of S3 bucket policy + + :return: S3 path (without bucket name) to saved object with elasticsearch domain policy backup + """ + timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds') + path = (f"queue_policies/" + f"{self.account.id}/" + f"{self.backup_filename.stem}_{timestamp}" + f"{self.backup_filename.suffix}") + if S3Operations.object_exists(s3_client, bucket, path): + raise Exception(f"s3://{bucket}/{path} already exists") + S3Operations.put_object(s3_client, bucket, path, self.policy) + return path + + def restrict_policy(self): + """ + Restrict and replace current policy on domain. + + :return: nothing + + .. note:: This keeps self._policy unchanged. + You need to recheck Elasticsearch domain policy to ensure that it was really restricted. + """ + restricted_policy = {} + policy_statement = {} + principal = {} + statement = [] + + principal["AWS"] = "*" + policy_statement["Effect"] = "Deny" + policy_statement["Principal"] = principal + policy_statement["Action"] = "es*" + policy_statement["Resource"] = self.arn + "/*" + statement.append(policy_statement) + restricted_policy["Statement"] = statement + + try: + ElasticSearchOperations.put_domain_policy(self.account.client("es"), self.name, restricted_policy) + except Exception: + logging.exception(f"Failed to put {self.name} restricted policy") + return False + + return True + + def set_logging(self): + """ + + :return: + """ + try: + ElasticSearchOperations.set_domain_logging(self.account.client("es"), self.account.client("logs"), + self.name) + except Exception: + logging.exception(f"Failed to enable {self.name} logging") + return False + + return True + + +class ESDomainChecker: + """ + Basic class for checking Elasticsearch unencrypted and logging issues in account/region. + Encapsulates discovered Elasticsearch domains. + """ + + def __init__(self, account): + """ + :param account: `Account` instance with Elasticsearch domains to check + """ + self.account = account + self.domains = [] + + def get_domain(self, id): + """ + :return: `Elasticsearch Domain` by id + """ + for domain in self.domains: + if domain.name == id: + return domain + return None + + def check(self, ids=None): + """ + Walk through Elasticsearch domains in the account/region and put them to `self.domains`. + + :param ids: list with Elasticsearch domain ids to check, + if it is not supplied - all Elasticsearch domains must be checked + + :return: boolean. True - if check was successful, + False - otherwise + """ + domain_details = [] + try: + es_client = self.account.client("es") + if ids is None: + ids = [] + domain_names_list = es_client.list_domain_names()["DomainNames"] + for domain_name in domain_names_list: + ids.append(domain_name["DomainName"]) + + if ids is not None: + domain_details = es_client.describe_elasticsearch_domains(DomainNames=ids)["DomainStatusList"] + + except ClientError as err: + if err.response['Error']['Code'] in ["AccessDenied", "AccessDeniedException", "UnauthorizedOperation"]: + logging.error(f"Access denied in {self.account} " + f"(ec2:{err.operation_name})") + else: + logging.error(f"Failed to describe elasticsearch domains in {self.account}") + return False + + for domain_detail in domain_details: + is_logging = False + domain_encrypted_at_rest = False + domain_encrypted_at_transit = False + domain_name = domain_detail["DomainName"] + domain_id = domain_detail["DomainId"] + domain_arn = domain_detail["ARN"] + encryption_at_rest = domain_detail.get("EncryptionAtRestOptions") + node_to_node_encryption = domain_detail.get("NodeToNodeEncryptionOptions") + if encryption_at_rest and encryption_at_rest["Enabled"]: + domain_encrypted_at_rest = True + if node_to_node_encryption and node_to_node_encryption["Enabled"]: + domain_encrypted_at_transit = True + + logging_details = domain_detail.get("LogPublishingOptions") + + if logging_details: + index_logs = logging_details.get("INDEX_SLOW_LOGS") + search_logs = logging_details.get("SEARCH_SLOW_LOGS") + error_logs = logging_details.get("ES_APPLICATION_LOGS") + if (index_logs and index_logs["Enabled"]) \ + or (search_logs and search_logs["Enabled"]) \ + or (error_logs and error_logs["Enabled"]): + is_logging = True + + tags = es_client.list_tags(ARN=domain_arn)["TagList"] + + access_policy = domain_detail.get("AccessPolicies") + + domain = ESDomainDetails(self.account, + name=domain_name, + id=domain_id, + arn=domain_arn, + tags=tags, + is_logging=is_logging, + encrypted_at_rest=domain_encrypted_at_rest, + encrypted_at_transit=domain_encrypted_at_transit, + policy=access_policy) + self.domains.append(domain) + return True \ No newline at end of file diff --git a/hammer/library/aws/iam.py b/hammer/library/aws/iam.py index 9c018c4b..d7960ba9 100755 --- a/hammer/library/aws/iam.py +++ b/hammer/library/aws/iam.py @@ -308,9 +308,12 @@ def check(self, users_to_check=None, last_used_check_enabled=False): :return: boolean. True - if check was successful, False - otherwise """ + users = [] try: # get all users in account - users = self.account.client("iam").list_users()['Users'] + paginator = self.account.client("iam").get_paginator("list_users") + for page in paginator.paginate(): + users.extend(page['Users']) except ClientError as err: if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: logging.error(f"Access denied in {self.account} " diff --git a/hammer/library/aws/rds.py b/hammer/library/aws/rds.py index 7a40c365..0d81a6b3 100755 --- a/hammer/library/aws/rds.py +++ b/hammer/library/aws/rds.py @@ -36,8 +36,11 @@ def get_rds_instance_details_of_sg_associated(cls, rds_client, group_id): rds_instances = [] # this will include both DB and Cluster instances - rds_response = rds_client.describe_db_instances() - for db_instance in rds_response["DBInstances"]: + rds_response = [] + paginator = rds_client.get_paginator('describe_db_instances') + for page in paginator.paginate(**args): + rds_response.extend(page["DBInstances"]) + for db_instance in rds_response: active_security_groups = [ sg["VpcSecurityGroupId"] for sg in db_instance['VpcSecurityGroups'] if sg["Status"] == "active" ] if group_id in active_security_groups: rds_instances.append(RDSInstance( diff --git a/hammer/library/aws/redshift.py b/hammer/library/aws/redshift.py new file mode 100644 index 00000000..9078c6a4 --- /dev/null +++ b/hammer/library/aws/redshift.py @@ -0,0 +1,199 @@ +import logging + +from botocore.exceptions import ClientError +from library.utility import timeit +from collections import namedtuple +from library.aws.utility import convert_tags + +# structure which describes EC2 instance +RedshiftCluster_Details = namedtuple('RedshiftCluster_Details', [ + # cluster_id + 'id', + # subnet_group_id + 'subnet_group_name' +]) + + +class RedshiftClusterOperations(object): + @classmethod + @timeit + def get_redshift_vpc_security_groups(cls, redshift_client, group_id): + """ Retrieve redshift clusters meta data with security group attached + + :param redshift_client: boto3 redshift client + :param group_id: security group id + + :return: list with redshift clusters details + """ + # describe rds instances with security group attached + redshift_clusters = [] + + # this will include Clusters + clusters_res = redshift_client.describe_clusters() + for cluster in clusters_res["Clusters"]: + active_security_groups = [sg["VpcSecurityGroupId"] for sg in cluster['VpcSecurityGroups'] if + sg["Status"] == "active"] + if group_id in active_security_groups: + redshift_clusters.append(RedshiftCluster_Details( + id=cluster["ClusterIdentifier"], + subnet_group_name=cluster["ClusterSubnetGroupName"] + )) + + return redshift_clusters + + @staticmethod + def make_private(redshift_client, cluster_id): + """ + Sets the cluster access as private. + + :param redshift_client: Redshift boto3 client + :param cluster_id: Redshift cluster name which to make as private. + + :return: nothing + """ + + redshift_client.modify_cluster( + ClusterIdentifier=cluster_id, + PubliclyAccessible=False + ) + + @staticmethod + def cluster_encryption(redshift_client, cluster_id): + """ + :param redshift_client: redshift client + :param cluster_id: cluster id which need to be encrypted. + + :return: + """ + # Modify cluster as encrypted. + redshift_client.modify_cluster( + ClusterIdentifier=cluster_id, + Encrypted=True + ) + + +class RedshiftCluster(object): + """ + Basic class for Redshift Cluster. + Encapsulates `Owner`/`Tags`. + """ + def __init__(self, account, name, tags, is_encrypted=None, is_public=None, is_logging=None): + """ + :param account: `Account` instance where redshift cluster is present + + :param name: `Name` of cluster id + :param tags: tags if redshift cluster tags (as AWS returns) + :param is_encrypted: encrypted or not. + """ + self.account = account + self.name = name + self.tags = convert_tags(tags) + self.is_encrypt = is_encrypted + self.is_public = is_public + self.is_logging = is_logging + + def make_private(self): + """ + Modify cluster as private. + :return: nothing + """ + try: + RedshiftClusterOperations.make_private(self.account.client("redshift"), self.name) + except Exception: + logging.exception(f"Failed to modify {self.name} cluster ") + return False + + return True + + def encrypt_cluster(self): + """ + Modify cluster as encrypted. + :return: nothing + """ + try: + RedshiftClusterOperations.cluster_encryption(self.account.client("redshift"), self.name) + except Exception: + logging.exception(f"Failed to modify {self.name} cluster encryption ") + return False + + return True + + +class RedshiftClusterChecker(object): + """ + Basic class for checking redshift clusters public access and encryption in account/region. + Encapsulates check settings and discovered clusters. + """ + + def __init__(self, account): + """ + :param account: `Account` clusters to check + + """ + self.account = account + self.clusters = [] + + def get_cluster(self, name): + """ + :return: `Redshift cluster` by name + """ + for cluster in self.clusters: + if cluster.name == name: + return cluster + return None + + def check(self, clusters=None): + """ + Walk through clusters in the account/region and check them. + Put all gathered clusters to `self.clusters`. + + :param clusters: list with clusters to check, if it is not supplied - all clusters must be checked + + :return: boolean. True - if check was successful, + False - otherwise + """ + try: + # AWS does not support filtering dirung list, so get all clusters for account + response = self.account.client("redshift").describe_clusters() + except ClientError as err: + if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: + logging.error(f"Access denied in {self.account} " + f"(redshift:{err.operation_name})") + else: + logging.exception(f"Failed to list cluster in {self.account}") + return False + + if "Clusters" in response: + for cluster_details in response["Clusters"]: + logging_enabled = False + tags = {} + cluster_id = cluster_details["ClusterIdentifier"] + + if clusters is not None and cluster_id not in clusters: + continue + + is_public = cluster_details["PubliclyAccessible"] + is_encrypted = cluster_details["Encrypted"] + if "Tags" in cluster_details: + tags = cluster_details["Tags"] + try: + logging_details = self.account.client("redshift").describe_logging_status( + ClusterIdentifier=cluster_id) + if "LoggingEnabled" in logging_details: + logging_enabled = logging_details["LoggingEnabled"] + except ClientError as err: + if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: + logging.error(f"Access denied in {self.account} " + f"(redshift:{err.operation_name})") + else: + logging.exception(f"Failed to describe logging status cluster in {self.account}") + + cluster = RedshiftCluster(account=self.account, + name=cluster_id, + tags=tags, + is_encrypted=is_encrypted, + is_public=is_public, + is_logging=logging_enabled) + self.clusters.append(cluster) + + return True diff --git a/hammer/library/aws/security_groups.py b/hammer/library/aws/security_groups.py index 814d88aa..f284b7b7 100755 --- a/hammer/library/aws/security_groups.py +++ b/hammer/library/aws/security_groups.py @@ -8,12 +8,15 @@ from library.utility import jsonDumps from library.aws.s3 import S3Operations from library.aws.utility import convert_tags +from library.config import Config +from library.utility import get_registrant class RestrictionStatus(Enum): Restricted = "restricted" OpenCompletely = "open_completely" OpenPartly = "open_partly" + ExcludedRegistrant = 'owner' class SecurityGroupOperations: @@ -65,7 +68,7 @@ def find_source_s3(account, if objects is None: logging.error(f"Failed to find '{group_id}' rules backup in {account}") return - backup_objects = [ obj["Key"] for obj in objects if obj.get("Key", "").startswith(f"{prefix}{group_id}_") ] + backup_objects = [obj["Key"] for obj in objects if obj.get("Key", "").startswith(f"{prefix}{group_id}_")] # return most recent backup recent_backup = max(backup_objects) source = json.loads(S3Operations.get_object(s3_client, bucket, recent_backup)) @@ -99,8 +102,8 @@ def restore_s3(cls, from_port = ingress.get("FromPort", None) to_port = ingress.get("ToPort", None) ip_protocol = ingress["IpProtocol"] - cidrs = [ ipv6_range["CidrIpv6"] for ipv6_range in ingress.get("Ipv6Ranges", []) ] - cidrs += [ ip_range["CidrIp"] for ip_range in ingress.get("IpRanges", []) ] + cidrs = [ipv6_range["CidrIpv6"] for ipv6_range in ingress.get("Ipv6Ranges", [])] + cidrs += [ip_range["CidrIp"] for ip_range in ingress.get("IpRanges", [])] for cidr in cidrs: cls.add_inbound_rule(ec2_client, group_id, ip_protocol, from_port, to_port, cidr) @@ -117,9 +120,9 @@ def ip_permissions(ip_protocol, from_port, to_port, cidr): :return: dict with `IpPermissions` element """ - perms = { 'IpProtocol': ip_protocol } + perms = {'IpProtocol': ip_protocol} if from_port is not None and \ - to_port is not None: + to_port is not None: perms['FromPort'] = from_port perms['ToPort'] = to_port ipv = ipaddress.ip_network(cidr).version @@ -372,6 +375,24 @@ def __str__(self): perms = ", ".join([str(perm) for perm in self.permissions]) return f"{self.__class__.__name__}(Name={self.name}, Id={self.id}, Permissions=[{perms}])" + @staticmethod + def validate_trusted_registrant(cidr): + """ + :param cidr: + :return: + """ + trusted_registrants = Config().sg.trusted_registrants + + if not trusted_registrants: + return False + + registrant = get_registrant(cidr) + + if registrant and (registrant['name'] in trusted_registrants + or registrant['title'] in trusted_registrants): + return True + return False + def restriction_status(self, cidr): """ Check restriction status of cidr @@ -381,7 +402,9 @@ def restriction_status(self, cidr): :return: RestrictionStatus with check result """ status = RestrictionStatus.Restricted - if cidr.endswith("/0"): + if ipaddress.ip_network(cidr).is_global and self.validate_trusted_registrant(cidr): + status = RestrictionStatus.ExcludedRegistrant + elif cidr.endswith("/0"): status = RestrictionStatus.OpenCompletely elif ipaddress.ip_network(cidr).is_global: status = RestrictionStatus.OpenPartly @@ -409,6 +432,9 @@ def check(self, restricted_ports): if status == RestrictionStatus.Restricted: logging.debug(f"Skipping restricted '{ip_range}'") continue + elif status == RestrictionStatus.ExcludedRegistrant: + logging.debug(f"Skipping excluded '{ip_range}'") + continue # second - check if ports from `restricted_ports` list has intersection with ports from FromPort..ToPort range if perm.from_port is None or perm.to_port is None: logging.debug(f"Marking world-wide open all ports from '{ip_range}'") @@ -432,6 +458,8 @@ def status(self): statuses = {perms.status for perms in self.permissions} if RestrictionStatus.OpenCompletely in statuses: return RestrictionStatus.OpenCompletely + elif RestrictionStatus.ExcludedRegistrant in statuses: + return RestrictionStatus.ExcludedRegistrant elif RestrictionStatus.OpenPartly in statuses: return RestrictionStatus.OpenPartly return RestrictionStatus.Restricted @@ -529,8 +557,11 @@ def check(self, ids=None, tags=None): args['Filters'].append( {'Name': f"tag:{key}", 'Values': value if isinstance(value, list) else [value]}, ) + secgroups = [] try: - secgroups = self.account.client("ec2").describe_security_groups(**args)["SecurityGroups"] + paginator = self.account.client("ec2").get_paginator('describe_security_groups') + for page in paginator.paginate(**args): + secgroups.extend(page["SecurityGroups"]) except ClientError as err: if err.response['Error']['Code'] in ["AccessDenied", "UnauthorizedOperation"]: logging.error(f"Access denied in {self.account} " diff --git a/hammer/library/config.py b/hammer/library/config.py index 504f1a1d..ec89c837 100755 --- a/hammer/library/config.py +++ b/hammer/library/config.py @@ -21,19 +21,23 @@ def __init__(self, configIniFile="config.ini", whitelistFile="whitelist.json", fixnowFile="fixnow.json", - ticketOwnersFile="ticket_owners.json"): + ticketOwnersFile="ticket_owners.json", + tempWhitelistFile="temp_whitelist_issues.json"): """ :param configFile: local path to configuration file in json format :param configIniFile: local path to configuration file in ini format (is used in r&r EC2, build from EC2 userdata) :param whitelistFile: local path to whitelist file in json format :param fixnowFile: local path to fixnow file in json format :param ticketOwnersFile: local path to file with default ticket owners by bu/account in json format + :param tempWhitelistFile: local path to list of temporary whitelist issues file in json format """ self._config = self.json_load_from_file(configFile) self._config['whitelist'] = self.json_load_from_file(whitelistFile, default={}) self._config['fixnow'] = self.json_load_from_file(fixnowFile, default={}) + self._config['tempwhitelist'] = self.json_load_from_file(tempWhitelistFile, default={}) + self.local = LocalConfig(configIniFile) self.owners = OwnersConfig(self.json_load_from_file(ticketOwnersFile, default={})) self.cronjobs = self._config.get('cronjobs', {}) @@ -63,8 +67,28 @@ def __init__(self, # RDS encryption issue config self.rdsEncrypt = ModuleConfig(self._config, "rds_encryption") + self.redshift_public_access = ModuleConfig(self._config, "redshift_public_access") + self.redshiftEncrypt = ModuleConfig(self._config, "redshift_encryption") + self.redshift_logging = ModuleConfig(self._config, "redshift_logging") # AMI public access issue config self.publicAMIs = ModuleConfig(self._config, "ec2_public_ami") + # ECS logging issue config + self.ecs_logging = ModuleConfig(self._config, "ecs_logging") + + # ECS access issue config + self.ecs_privileged_access = ModuleConfig(self._config, "ecs_privileged_access") + + # ECS image source issue config + self.ecs_external_image_source = ModuleConfig(self._config, "ecs_external_image_source") + + # Elasticsearch domain logging issue config + self.esLogging = ModuleConfig(self._config, "es_domain_logging") + + # Elasticsearch unencrypted domain issue config + self.esEncrypt = ModuleConfig(self._config, "es_unencrypted_domain") + + # Elasticsearch publicly accessed domain issue config + self.esPublicAccess = ModuleConfig(self._config, "es_public_access_domain") self.bu_list = self._config.get("bu_list", []) self.whitelisting_procedure_url = self._config.get("whitelisting_procedure_url", None) @@ -460,6 +484,7 @@ def __init__(self, config, section): super().__init__(config, section) self._whitelist = config["whitelist"].get(section, {}) self._fixnow = config["fixnow"].get(section, {}) + self._tempwhitelist_list = config["tempwhitelist"].get(section, {}) # main accounts dict self._accounts = config["aws"]["accounts"] self.name = section @@ -473,12 +498,13 @@ def module_accounts(self, option): :return: dict with AWS accounts to identify/remediate {'account id': 'account name', ...} """ module_accounts = self._config.get(option, None) - if module_accounts is None: + if module_accounts is None or len(module_accounts) == 0: accounts = self._accounts else: # construct dict similar to main accounts dict accounts = {account: self._accounts.get(account, "") for account in module_accounts} # exclude 'ignore_accounts' from resulting dict + return {k: v for k, v in accounts.items() if k not in self._config.get("ignore_accounts", [])} @property @@ -522,6 +548,17 @@ def in_whitelist(self, account_id, issue): """ return issue in self._whitelist.get(account_id, []) + def in_temp_whitelist(self, account_id, issue): + """ + :param account_id: AWS account Id + :param issue: Issue id + + :return: boolean, if issue Id in temp whitelist file + """ + return issue in self._tempwhitelist_list.get(account_id, []) + + + @property def ddb_table_name(self): """ :return: DDB table name to use for storing issue details """ @@ -551,6 +588,11 @@ def issue_retention_date(self): """ :return: `timedelta` object before performing auto remediation """ return timedelta(days=self.remediation_retention_period) + @property + def trusted_registrants(self): + """ :return: list of trusted registrants""" + return self._config.get('trusted_registrants', []) + class IAMUserInactiveKeysConfig(ModuleConfig): """ Extend ModuleConfig with IAM inactive keys specific details """ diff --git a/hammer/library/ddb_issues.py b/hammer/library/ddb_issues.py index d9ae7de2..a1cbd6ea 100755 --- a/hammer/library/ddb_issues.py +++ b/hammer/library/ddb_issues.py @@ -20,6 +20,8 @@ class IssueStatus(Enum): Resolved = "resolved" # set by reporting after closing ticket Closed = "closed" + # set by identification - issue still exists but was added to temporary whitelist_list for future remediation + Tempwhitelist = "tempwhitelist" class Details(object): @@ -133,6 +135,8 @@ def from_dict(item, issue_class=None): def contains_tags(self, tags): if not tags: return True + if not self.issue_details.tags: + return False for k in tags: if k not in self.issue_details.tags: return False @@ -234,10 +238,55 @@ def __init__(self, *args): class PublicAMIIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class RedshiftEncryptionIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class RedshiftLoggingIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class RedshiftPublicAccessIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class ECSLoggingIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class ECSPrivilegedAccessIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class ECSExternalImageSourceIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class ESEncryptionIssue(Issue): def __init__(self, *args): super().__init__(*args) + +class ESLoggingIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + + +class ESPublicAccessIssue(Issue): + def __init__(self, *args): + super().__init__(*args) + class Operations(object): @staticmethod def find(ddb_table, issue): @@ -431,3 +480,16 @@ def set_status_updated(cls, ddb_table, issue): """ issue.timestamps.updated = issue.timestamps.reported cls.put(ddb_table, issue) + + @classmethod + def set_status_temp_whitelisted(cls, ddb_table, issue): + """ + Put issue with closed status and updated closed timestamp + + :param ddb_table: boto3 DDB table resource + :param issue: Issue instance + + :return: nothing + """ + issue.timestamps.temp_whitelisted = datetime.now(timezone.utc).isoformat() + cls.put(ddb_table, issue) \ No newline at end of file diff --git a/hammer/library/jiraoperations.py b/hammer/library/jiraoperations.py index cc71c00e..c45aeac7 100755 --- a/hammer/library/jiraoperations.py +++ b/hammer/library/jiraoperations.py @@ -2,35 +2,80 @@ import logging import urllib3 - from collections import namedtuple from jira import JIRA from jira import JIRAError from library.utility import empty_converter - NewIssue = namedtuple('NewIssue', [ 'ticket_id', 'ticket_assignee_id' - ]) +]) + + +class JiraLabels(object): + """ Base class for JIRA tickets labeling """ + DEFAULT_LABELS = { + 'cloudtrails': ['cloudtrail-issue'], + 'ebsSnapshot': ['ebs-public-snapshot'], + 'ebsVolume': ['ebs-unencrypted-volume'], + 'ecsExternalImageSource': ['ecs-external-image'], + 'ecsLogging': ['ecs-logging'], + 'ecsPrivilegedAccess': ['ecs-privileged-access'], + 'esDomainLogging': ['es-domain-logging'], + 'esPublicAccessDomain': ['es-public-access-domain'], + 'esUnencryptedDomain': ['es-unencrypted-domain'], + 'iamUserInactiveKeys': ['iam-key-inactive'], + 'iamUserKeysRotation': ['iam-key-rotation'], + 'publicAMIs': ['public-ami'], + 'rdsSnapshot': ['rds-public-snapshot'], + 'rdsEncrypt': ['rds-unencrypted'], + 'redshiftLogging': ['redshift-logging'], + 'redshiftPublicAccess': ['redshift-public-access'], + 'redshiftUnencrypted': ['redshift-encryption'], + 's3Encrypt': ['s3-unencrypted'], + 's3acl': ['s3-public-acl'], + 's3policy': ['s3-public-policy'], + 'sg': ['insecure-services'], + 'sqspolicy': ['sqs-public-policy'] + } + + def __init__(self, config, module=''): + self.config = config + self.module = module + self.module_jira = getattr(config, module) if hasattr(config, module) else False + self.module_jira_labels = self.module_jira.labels if hasattr(self.module_jira, 'labels') else False + + @property + def module_labels(self): + if self.module_jira_labels: + return self.module_jira_labels + else: + return self.DEFAULT_LABELS.get(self.module, '') class JiraReporting(object): """ Base class for JIRA reporting """ - def __init__(self, config): + def __init__(self, config, module=''): self.config = config self.jira = JiraOperations(self.config) + self.jira_labels = JiraLabels(config, module) + self.module_jira_labels = self.jira_labels.module_labels + def _jira_enabled(func): + def decorated(self, *args, **kwargs): + if self.config.jira.enabled: + return func(self, *args, **kwargs) + return decorated + + @_jira_enabled def add_issue(self, issue_summary, issue_description, - priority, labels, + priority, account_id, owner=None, bu=None, product=None, ): - # TODO: move to decorator - if not self.config.jira.enabled: - return None project = self.config.owners.ticket_project( bu=bu, product=product, @@ -43,7 +88,7 @@ def add_issue(self, "description": issue_description, "issuetype": {"name": self.config.jira.issue_type}, "priority": {"name": priority}, - "labels": labels + "labels": self.module_jira_labels } ticket_id = self.jira.create_ticket(issue_data) @@ -74,36 +119,24 @@ def add_issue(self, return NewIssue(ticket_id=ticket_id, ticket_assignee_id=ticket_assignee_id) + @_jira_enabled def close_issue(self, ticket_id, comment): - # TODO: move to decorator - if not self.config.jira.enabled: - return - self.jira.add_comment(ticket_id, comment) self.jira.close_issue(ticket_id) logging.debug(f"Closed issue ({self.jira.ticket_url(ticket_id)})") + @_jira_enabled def update_issue(self, ticket_id, comment): - # TODO: move to decorator - if not self.config.jira.enabled: - return - # TODO: reopen ticket if closed self.jira.add_comment(ticket_id, comment) logging.debug(f"Updated issue {self.jira.ticket_url(ticket_id)}") + @_jira_enabled def add_attachment(self, ticket_id, filename, text): - # TODO: move to decorator - if not self.config.jira.enabled: - return - return self.jira.add_attachment(ticket_id, filename, text) + @_jira_enabled def remediate_issue(self, ticket_id, comment, reassign): - # TODO: move to decorator - if not self.config.jira.enabled: - return - if reassign: self.jira.assign_user(ticket_id, self.jira.current_user) self.jira.add_comment(ticket_id, comment) @@ -114,6 +147,7 @@ def ticket_url(self, ticket_id): def add_label(self, ticket_id, label): self.jira.add_label(ticket_id, label) + class JiraOperations(object): """ Base class for interaction with JIRA """ def __init__(self, config): @@ -330,7 +364,7 @@ def add_comment(self, ticket_id, comment): def add_watcher(self, ticket_id, user): """ Adding jira ticket watcher. - + :param ticket_id: jira ticket id :param user: watcher user id :return: nothing diff --git a/hammer/library/slack_utility.py b/hammer/library/slack_utility.py index a9bc179a..8b54d497 100755 --- a/hammer/library/slack_utility.py +++ b/hammer/library/slack_utility.py @@ -52,8 +52,8 @@ def post_message(self, msg, owner=None): # if owner is not set - try to find channel to send msg to based on msg body owner = owner if owner is not None else self.config.slack.find_channel(msg) - # open user channel if owner is not prefixed with # - channel = owner if owner.startswith("#") else self.open_user_channel(owner) + # get user id if owner is not prefixed with # + channel = owner if owner.startswith("#") else self.user_id(owner) if not channel: logging.debug(f"ignoring: '{msg}'") diff --git a/hammer/library/utility.py b/hammer/library/utility.py index 77e5689e..d136d4e3 100755 --- a/hammer/library/utility.py +++ b/hammer/library/utility.py @@ -1,5 +1,6 @@ import os import json +import warnings import xml import time import logging @@ -8,9 +9,10 @@ import tempfile import fcntl - from datetime import datetime from decimal import Decimal +from functools import lru_cache +from ipwhois import IPWhois def jsonEncoder(obj): @@ -120,6 +122,30 @@ def confirm(question, default=None): print("Please respond with 'yes' or 'no'") +@lru_cache(maxsize=128) +def get_registrant(cidr): + ip = cidr.split("/")[0] + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + whois = IPWhois(ip).lookup_rdap(asn_methods=['dns', 'whois', 'http']) + except Exception: + return "" + + registrant = {} + + for title, obj in whois.get('objects', {}).items(): + if obj.get('contact') is None: + continue + if 'registrant' in obj.get('roles', []): + registrant['name'] = obj['contact'].get('name') + registrant['title'] = title + break + + return registrant + + class SingletonInstanceException(BaseException): pass @@ -138,4 +164,4 @@ def __init__(self, instance_id): try: fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: - raise SingletonInstanceException() \ No newline at end of file + raise SingletonInstanceException() diff --git a/hammer/reporting-remediation/analytics/security_issues_csv_report.py b/hammer/reporting-remediation/analytics/security_issues_csv_report.py index 321eeee9..e6f85ae3 100755 --- a/hammer/reporting-remediation/analytics/security_issues_csv_report.py +++ b/hammer/reporting-remediation/analytics/security_issues_csv_report.py @@ -8,7 +8,10 @@ from library.aws.utility import AssumeRole from library.config import Config from library.ddb_issues import Operations as IssueOperations -from library.ddb_issues import SecurityGroupIssue, S3AclIssue, S3PolicyIssue, CloudTrailIssue, IAMKeyRotationIssue, IAMKeyInactiveIssue, RdsPublicSnapshotIssue, EBSUnencryptedVolumeIssue, EBSPublicSnapshotIssue, SQSPolicyIssue +from library.ddb_issues import SecurityGroupIssue, S3AclIssue, S3PolicyIssue, CloudTrailIssue, IAMKeyRotationIssue, \ + IAMKeyInactiveIssue, RdsPublicSnapshotIssue, EBSUnencryptedVolumeIssue, EBSPublicSnapshotIssue, SQSPolicyIssue,\ + RedshiftEncryptionIssue, RedshiftLoggingIssue, ECSPrivilegedAccessIssue, ECSLoggingIssue,\ + ECSExternalImageSourceIssue, RedshiftPublicAccessIssue, ESLoggingIssue, ESEncryptionIssue, ESPublicAccessIssue from analytics.add_excel_sheet_records import AddRecordsToSheet from library.slack_utility import SlackNotification from library.aws.s3 import S3Operations @@ -70,6 +73,16 @@ def generate(self): (self.config.cloudtrails.ddb_table_name, "CloudTrail Logging Issues", CloudTrailIssue), (self.config.rdsSnapshot.ddb_table_name, "RDS Public Snapshots", RdsPublicSnapshotIssue), (self.config.sqspolicy.ddb_table_name, "SQS Policy Public Access", SQSPolicyIssue), + (self.config.redshiftEncrypt.ddb_table_name, "Redshift Unencrypted Clusters", RedshiftEncryptionIssue), + (self.config.redshift_logging.ddb_table_name, "Redshift Logging Issues ", RedshiftLoggingIssue), + (self.config.redshift_public_access.ddb_table_name, "Redshift Public Clusters", RedshiftPublicAccessIssue), + (self.config.ecs_privileged_access.ddb_table_name, "ECS Privileged Access Issues", ECSPrivilegedAccessIssue), + (self.config.ecs_logging.ddb_table_name, "ECS Logging Issues", ECSLoggingIssue), + (self.config.ecs_external_image_source.ddb_table_name, "ECS External Image Sources", + ECSExternalImageSourceIssue), + (self.config.esLogging.ddb_table_name, "Elasticsearch Logging Issues", ESLoggingIssue), + (self.config.esEncrypt.ddb_table_name, "Elasticsearch Encryption Issue", ESEncryptionIssue), + (self.config.esPublicAccess.ddb_table_name, "Elasticsearch Public Access", ESPublicAccessIssue) ] open_security_issues_workbook = xlwt.Workbook() @@ -125,7 +138,6 @@ def generate(self): channel=channel) - if __name__ == '__main__': module_name = sys.modules[__name__].__loader__.name set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") diff --git a/hammer/reporting-remediation/bot/commands.py b/hammer/reporting-remediation/bot/commands.py index 7d79a24c..f1f79ab1 100644 --- a/hammer/reporting-remediation/bot/commands.py +++ b/hammer/reporting-remediation/bot/commands.py @@ -81,7 +81,15 @@ def status(message): else: response += f"`disabled`" response += "\n" - message.reply(response) + # FIXME this is workaround to issues "lost websocket connection, try to reconnect now" that + # happens after we send accounts' statuses to slack. The message is long, however slack should handle + # this case and truncate the message without closing connection. I limited the size of message to 15000 + # which seems to work. + msg_length = 15000 + msg_start = 0 + while msg_start < len(response): + message.reply(response[msg_start:msg_start + msg_length]) + msg_start = msg_start + msg_length @respond_to('^(?P
.*) config$', re.IGNORECASE) diff --git a/hammer/reporting-remediation/cronjobs/automation_scheduler.py b/hammer/reporting-remediation/cronjobs/automation_scheduler.py index f473312f..697f3619 100755 --- a/hammer/reporting-remediation/cronjobs/automation_scheduler.py +++ b/hammer/reporting-remediation/cronjobs/automation_scheduler.py @@ -44,19 +44,44 @@ def automation_cronjob(config): ) modules = [ - ("S3 ACL Public Access", config.s3acl, "create_s3bucket_acl_issue_tickets", "clean_s3bucket_acl_permissions"), - ("Insecure Services", config.sg, "create_security_groups_tickets", "clean_security_groups"), - ("IAM User Inactive Keys", config.iamUserInactiveKeys, "create_iam_key_inactive_tickets", "clean_iam_keys_inactive"), - ("IAM User Key Rotation", config.iamUserKeysRotation, "create_iam_key_rotation_tickets", "clean_iam_key_rotation"), - ("S3 Policy Public Access", config.s3policy, "create_s3bucket_policy_issue_tickets", "clean_s3bucket_policy_permissions"), + ("S3 ACL Public Access", config.s3acl, "create_s3bucket_acl_issue_tickets", + "clean_s3bucket_acl_permissions"), + ("Insecure Services", config.sg, "create_security_groups_tickets", + "clean_security_groups"), + ("IAM User Inactive Keys", config.iamUserInactiveKeys, "create_iam_key_inactive_tickets", + "clean_iam_keys_inactive"), + ("IAM User Key Rotation", config.iamUserKeysRotation, "create_iam_key_rotation_tickets", + "clean_iam_key_rotation"), + ("S3 Policy Public Access", config.s3policy, "create_s3bucket_policy_issue_tickets", + "clean_s3bucket_policy_permissions"), ("CloudTrail Logging Issues", config.cloudtrails, "create_cloudtrail_tickets", None), ("EBS Unencrypted Volumes", config.ebsVolume, "create_ebs_volume_issue_tickets", None), - ("EBS Public Snapshots", config.ebsSnapshot, "create_ebs_public_snapshot_issue_tickets", "clean_public_ebs_snapshots"), - ("RDS Public Snapshots", config.rdsSnapshot, "create_rds_public_snapshot_issue_tickets", "clean_public_rds_snapshots"), - ("EC2 Public Images", config.publicAMIs, "create_public_ami_issue_tickets", "clean_ami_public_access"), - ("SQS Public Access", config.sqspolicy, "create_sqs_policy_issue_tickets", "clean_sqs_policy_permissions"), - ("S3 Unencrypted Buckets", config.s3Encrypt, "create_s3_unencrypted_bucket_issue_tickets", "clean_s3bucket_unencrypted"), - ("RDS Unencrypted Instances", config.rdsEncrypt, "create_rds_unencrypted_instance_issue_tickets", None), + ("EBS Public Snapshots", config.ebsSnapshot, "create_ebs_public_snapshot_issue_tickets", + "clean_public_ebs_snapshots"), + ("RDS Public Snapshots", config.rdsSnapshot, "create_rds_public_snapshot_issue_tickets", + "clean_public_rds_snapshots"), + ("EC2 Public Images", config.publicAMIs, "create_public_ami_issue_tickets", + "clean_ami_public_access"), + ("SQS Public Access", config.sqspolicy, "create_sqs_policy_issue_tickets", + "clean_sqs_policy_permissions"), + ("S3 Unencrypted Buckets", config.s3Encrypt, "create_s3_unencrypted_bucket_issue_tickets", + "clean_s3bucket_unencrypted"), + ("RDS Unencrypted Instances", config.rdsEncrypt, "create_rds_unencrypted_instance_issue_tickets", None), + ("Redshift Public Clusters", config.redshift_public_access, "create_redshift_public_access_issue_tickets", + "clean_redshift_public_access"), + ("Redshift Unencrypted Clusters", config.redshiftEncrypt, "create_redshift_unencrypted_cluster_issue_tickets", + "clean_redshift_cluster_unencrypted"), + ("Redshift Logging Issues", config.redshift_logging, "create_redshift_logging_issue_tickets", None), + ("ECS Privileged Access Issues", config.ecs_privileged_access, "create_ecs_privileged_access_issue_tickets", + None), + ("ECS Logging Issues", config.ecs_logging, "create_ecs_logging_issue_tickets", None), + ("ECS External Image Sources", config.ecs_external_image_source, + "create_ecs_external_image_source_issue_tickets", None), + ("Elasticsearch Logging Issues", config.esLogging, "create_elasticsearch_domain_logging_issue_tickets", + "clean_elasticsearch_domain_logging"), + ("Elasticsearch Unencrypted Domains", config.esEncrypt, "create_elasticsearch_unencrypted_issue_tickets", None), + ("Elasticsearch Public Access", config.esPublicAccess, "create_elasticsearch_public_access_issue_tickets", + "clean_elasticsearch_policy_permissions") ] for title, module_config, reporting_script, remediation_script in modules: diff --git a/hammer/reporting-remediation/remediation/clean_ami_public_access.py b/hammer/reporting-remediation/remediation/clean_ami_public_access.py index 558f61c7..4145fe17 100644 --- a/hammer/reporting-remediation/remediation/clean_ami_public_access.py +++ b/hammer/reporting-remediation/remediation/clean_ami_public_access.py @@ -28,10 +28,10 @@ def clean_ami_public_access(self): retention_period = self.config.publicAMIs.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='publicAMIs') slack = SlackNotification(self.config) - for account_id, account_name in self.config.aws.accounts.items(): + for account_id, account_name in self.config.publicAMIs.remediation_accounts.items(): logging.debug(f"Checking '{account_name} / {account_id}'") issues = IssueOperations.get_account_open_issues(ddb_table, account_id, PublicAMIIssue) for issue in issues: @@ -39,6 +39,12 @@ def clean_ami_public_access(self): in_whitelist = self.config.publicAMIs.in_whitelist(account_id, ami_id) + in_temp_whitelist = self.config.publicAMIs.in_temp_whitelist(account_id, ami_id) + if in_temp_whitelist: + logging.debug(f"Skipping '{ami_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + if in_whitelist: logging.debug(f"Skipping {ami_id} (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_elasticsearch_domain_logging.py b/hammer/reporting-remediation/remediation/clean_elasticsearch_domain_logging.py new file mode 100644 index 00000000..417e4fc8 --- /dev/null +++ b/hammer/reporting-remediation/remediation/clean_elasticsearch_domain_logging.py @@ -0,0 +1,152 @@ +""" +Class to remediate ElasticSearch domain logging issues. +""" +import sys +import logging +import argparse + + +from library.logger import set_logging, add_cw_logging +from library.config import Config +from library.jiraoperations import JiraReporting +from library.slack_utility import SlackNotification +from library.ddb_issues import Operations as IssueOperations +from library.ddb_issues import IssueStatus, ESLoggingIssue +from library.aws.elasticsearch import ESDomainChecker +from library.aws.utility import Account +from library.utility import confirm +from library.utility import SingletonInstance, SingletonInstanceException + + +class CleanElasticSearchDomainLogging: + """ Class to remediate ElasticSearch domain logging issues """ + def __init__(self, config): + self.config = config + + def clean_elasticsearch_domain_domain_logging_issues(self, batch=False): + """ Class method to clean ElasticSearch domains which are violating aws best practices """ + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(self.config.esLogging.ddb_table_name) + backup_bucket = config.aws.s3_backup_bucket + + retention_period = self.config.esLogging.remediation_retention_period + + jira = JiraReporting(self.config, module='esDomainLogging') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.esLogging.remediation_accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ESLoggingIssue) + for issue in issues: + domain_name = issue.issue_id + + in_whitelist = self.config.esLogging.in_whitelist(account_id, domain_name) + in_temp_whitelist = self.config.esLogging.in_temp_whitelist(account_id, domain_name) + if in_temp_whitelist: + logging.debug(f"Skipping '{domain_name}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + + if in_whitelist: + logging.debug(f"Skipping {domain_name} (in whitelist)") + + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + continue + + if issue.timestamps.reported is None: + logging.debug(f"Skipping '{domain_name}' (was not reported)") + continue + + if issue.timestamps.remediated is not None: + logging.debug(f"Skipping {domain_name} (has been already remediated)") + continue + + updated_date = issue.timestamp_as_datetime + no_of_days_issue_created = (self.config.now - updated_date).days + + if no_of_days_issue_created >= retention_period: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + try: + account = Account(id=account_id, + name=account_name, + region=issue.issue_details.region, + role_name=self.config.aws.role_name_reporting) + if account.session is None: + continue + + checker = ESDomainChecker(account=account) + checker.check(ids=[domain_name]) + domain_details = checker.get_domain(domain_name) + if domain_details is None: + logging.debug(f"Elasticsearch domain {domain_name} was removed by user") + elif domain_details.is_logging: + logging.debug(f"Elasticsearch domain {domain_name} logging issue was remediated by user") + else: + if not batch and \ + not confirm(f"Do you want to remediate elasticsearch domain '{domain_name}' logging issue", False): + continue + + logging.debug(f"Remediating '{domain_name}' logging issue") + + remediation_succeed = True + if domain_details.set_logging(): + comment = (f"Domain '{domain_name}' logging issue " + f"in '{account_name} / {account_id}' account " + f"was remediated by hammer") + else: + remediation_succeed = False + comment = (f"Failed to remediate elasticsearch domain '{domain_name}' logging issue " + f"in '{account_name} / {account_id}' account " + f"due to some limitations. Please, check manually") + + jira.remediate_issue( + ticket_id=issue.jira_details.ticket, + comment=comment, + reassign=remediation_succeed, + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_remediated(ddb_table, issue) + except Exception: + logging.exception(f"Error occurred while updating domain '{domain_name}' logging " + f"in '{account_name} / {account_id}'") + else: + logging.debug(f"Skipping '{domain_name}' " + f"({retention_period - no_of_days_issue_created} days before remediation)") + + +if __name__ == "__main__": + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + parser = argparse.ArgumentParser() + parser.add_argument('--batch', action='store_true', help='Do not ask confirmation for remediation') + args = parser.parse_args() + + try: + class_object = CleanElasticSearchDomainLogging(config) + class_object.clean_elasticsearch_domain_domain_logging_issues(batch=args.batch) + except Exception: + logging.exception("Failed to clean Elasticsearch domain logging issues") diff --git a/hammer/reporting-remediation/remediation/clean_elasticsearch_policy_permissions.py b/hammer/reporting-remediation/remediation/clean_elasticsearch_policy_permissions.py new file mode 100644 index 00000000..c0fb51d8 --- /dev/null +++ b/hammer/reporting-remediation/remediation/clean_elasticsearch_policy_permissions.py @@ -0,0 +1,159 @@ +""" +Class to remediate ElasticSearch policy permissions. +""" +import sys +import logging +import argparse + + +from library.logger import set_logging, add_cw_logging +from library.config import Config +from library.jiraoperations import JiraReporting +from library.slack_utility import SlackNotification +from library.ddb_issues import Operations as IssueOperations +from library.ddb_issues import IssueStatus, ESPublicAccessIssue +from library.aws.elasticsearch import ESDomainChecker +from library.aws.utility import Account +from library.utility import confirm +from library.utility import SingletonInstance, SingletonInstanceException + + +class CleanElasticSearchPolicyPermissions: + """ Class to remediate ElasticSearch domain policy permissions """ + def __init__(self, config): + self.config = config + + def clean_elasticsearch_domain_policy_permissions(self, batch=False): + """ Class method to clean ElasticSearch domains which are violating aws best practices """ + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(self.config.esPublicAccess.ddb_table_name) + backup_bucket = config.aws.s3_backup_bucket + + retention_period = self.config.esPublicAccess.remediation_retention_period + + jira = JiraReporting(self.config, module='esPublicAccessDomain') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.esPublicAccess.remediation_accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_open_issues(ddb_table, account_id, ESPublicAccessIssue) + for issue in issues: + domain_name = issue.issue_id + + in_whitelist = self.config.esPublicAccess.in_whitelist(account_id, domain_name) + # in_fixlist = self.config.esPublicAccess.in_fixnow(account_id, domain_name) + in_temp_whitelist = self.config.esPublicAccess.in_temp_whitelist(account_id, domain_name) + if in_temp_whitelist: + logging.debug(f"Skipping '{domain_name}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + + if in_whitelist: + logging.debug(f"Skipping {domain_name} (in whitelist)") + + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + continue + # if not in_fixlist: + # logging.debug(f"Skipping {domain_name} (not in fixlist)") + # continue + + if issue.timestamps.reported is None: + logging.debug(f"Skipping '{domain_name}' (was not reported)") + continue + + if issue.timestamps.remediated is not None: + logging.debug(f"Skipping {domain_name} (has been already remediated)") + continue + + updated_date = issue.timestamp_as_datetime + no_of_days_issue_created = (self.config.now - updated_date).days + + if no_of_days_issue_created >= retention_period: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + try: + account = Account(id=account_id, + name=account_name, + region=issue.issue_details.region, + role_name=self.config.aws.role_name_reporting) + if account.session is None: + continue + + checker = ESDomainChecker(account=account) + checker.check(ids=[domain_name]) + domain_details = checker.get_domain(domain_name) + if domain_details is None: + logging.debug(f"Elasticsearch domain {domain_name} was removed by user") + elif not domain_details.public: + logging.debug(f"Elasticsearch domain {domain_name} policy issue was remediated by user") + else: + if not batch and \ + not confirm(f"Do you want to remediate elasticsearch domain '{domain_name}' policy", False): + continue + + logging.debug(f"Remediating '{domain_name}' policy") + + backup_path = domain_details.backup_policy_s3(main_account.client("s3"), backup_bucket) + remediation_succeed = True + if domain_details.restrict_policy(): + comment = (f"Policy backup was saved to " + f"[{backup_path}|https://s3.console.aws.amazon.com/s3/object/{backup_bucket}/{backup_path}]. " + f"Domain '{domain_name}' policy issue " + f"in '{account_name} / {account_id}' account " + f"was remediated by hammer") + else: + remediation_succeed = False + comment = (f"Failed to remediate elasticsearch domain '{domain_name}' policy issue " + f"in '{account_name} / {account_id}' account " + f"due to some limitations. Please, check manually") + + jira.remediate_issue( + ticket_id=issue.jira_details.ticket, + comment=comment, + reassign=remediation_succeed, + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_remediated(ddb_table, issue) + except Exception: + logging.exception(f"Error occurred while updating domain '{domain_name}' policy " + f"in '{account_name} / {account_id}'") + else: + logging.debug(f"Skipping '{domain_name}' " + f"({retention_period - no_of_days_issue_created} days before remediation)") + + +if __name__ == "__main__": + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + parser = argparse.ArgumentParser() + parser.add_argument('--batch', action='store_true', help='Do not ask confirmation for remediation') + args = parser.parse_args() + + try: + class_object = CleanElasticSearchPolicyPermissions(config) + class_object.clean_elasticsearch_domain_policy_permissions(batch=args.batch) + except Exception: + logging.exception("Failed to clean Elasticsearch domain public policies") diff --git a/hammer/reporting-remediation/remediation/clean_iam_key_rotation.py b/hammer/reporting-remediation/remediation/clean_iam_key_rotation.py index 42598920..b6f5ac46 100755 --- a/hammer/reporting-remediation/remediation/clean_iam_key_rotation.py +++ b/hammer/reporting-remediation/remediation/clean_iam_key_rotation.py @@ -30,7 +30,7 @@ def clean_iam_access_keys(self, batch=False): retention_period = self.config.iamUserKeysRotation.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='iamUserKeysRotation') slack = SlackNotification(self.config) for account_id, account_name in self.config.iamUserKeysRotation.remediation_accounts.items(): @@ -43,6 +43,13 @@ def clean_iam_access_keys(self, batch=False): user_in_whitelist = self.config.iamUserKeysRotation.in_whitelist(account_id, username) key_in_whitelist = self.config.iamUserKeysRotation.in_whitelist(account_id, key_id) + user_in_temp_whitelist = self.config.iamUserKeysRotation.in_temp_whitelist(account_id, username) + key_in_temp_whitelist = self.config.iamUserKeysRotation.in_temp_whitelist(account_id, key_id) + if user_in_temp_whitelist or key_in_temp_whitelist: + logging.debug(f"Skipping '{key_id} / {username}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + if user_in_whitelist or key_in_whitelist: logging.debug(f"Skipping '{key_id} / {username}' (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_iam_keys_inactive.py b/hammer/reporting-remediation/remediation/clean_iam_keys_inactive.py index 6969f360..2b20c108 100755 --- a/hammer/reporting-remediation/remediation/clean_iam_keys_inactive.py +++ b/hammer/reporting-remediation/remediation/clean_iam_keys_inactive.py @@ -30,7 +30,7 @@ def clean_iam_access_keys(self, batch=False): retention_period = self.config.iamUserInactiveKeys.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='iamUserInactiveKeys') slack = SlackNotification(self.config) for account_id, account_name in self.config.iamUserInactiveKeys.remediation_accounts.items(): @@ -42,6 +42,13 @@ def clean_iam_access_keys(self, batch=False): user_in_whitelist = self.config.iamUserInactiveKeys.in_whitelist(account_id, username) key_in_whitelist = self.config.iamUserInactiveKeys.in_whitelist(account_id, key_id) + user_in_temp_whitelist = self.config.iamUserInactiveKeys.in_temp_whitelist(account_id, username) + key_in_temp_whitelist = self.config.iamUserInactiveKeys.in_temp_whitelist(account_id, key_id) + if user_in_temp_whitelist or key_in_temp_whitelist: + logging.debug( + f"Skipping '{key_id} / {username}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue if user_in_whitelist or key_in_whitelist: logging.debug(f"Skipping '{key_id} / {username}' (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_public_ebs_snapshots.py b/hammer/reporting-remediation/remediation/clean_public_ebs_snapshots.py index 99df6d66..1df6d270 100755 --- a/hammer/reporting-remediation/remediation/clean_public_ebs_snapshots.py +++ b/hammer/reporting-remediation/remediation/clean_public_ebs_snapshots.py @@ -30,7 +30,7 @@ def clean_public_ebs_snapshots(self, batch=False): retention_period = self.config.ebsSnapshot.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='ebsSnapshot') slack = SlackNotification(self.config) for account_id, account_name in self.config.ebsSnapshot.remediation_accounts.items(): @@ -42,6 +42,12 @@ def clean_public_ebs_snapshots(self, batch=False): continue in_whitelist = self.config.ebsSnapshot.in_whitelist(account_id, issue.issue_id) + in_temp_whitelist = self.config.ebsSnapshot.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug(f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + if in_whitelist: logging.debug(f"Skipping '{issue.issue_id}' (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_public_rds_snapshots.py b/hammer/reporting-remediation/remediation/clean_public_rds_snapshots.py index 60511540..3430f3b3 100755 --- a/hammer/reporting-remediation/remediation/clean_public_rds_snapshots.py +++ b/hammer/reporting-remediation/remediation/clean_public_rds_snapshots.py @@ -31,7 +31,7 @@ def clean_public_rds_snapshots(self, batch=False): retention_period = self.config.rdsSnapshot.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='rdsSnapshot') slack = SlackNotification(self.config) for account_id, account_name in self.config.rdsSnapshot.remediation_accounts.items(): @@ -43,6 +43,13 @@ def clean_public_rds_snapshots(self, batch=False): continue in_whitelist = self.config.rdsSnapshot.in_whitelist(account_id, issue.issue_id) + in_temp_whitelist = self.config.rdsSnapshot.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + if in_whitelist: logging.debug(f"Skipping '{issue.issue_id}' (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_redshift_cluster_unencrypted.py b/hammer/reporting-remediation/remediation/clean_redshift_cluster_unencrypted.py new file mode 100644 index 00000000..1fc56225 --- /dev/null +++ b/hammer/reporting-remediation/remediation/clean_redshift_cluster_unencrypted.py @@ -0,0 +1,153 @@ +""" +Class to remediate Redshift cluster un-encryption issues. +""" +import sys +import logging +import argparse + + +from library.logger import set_logging, add_cw_logging +from library.config import Config +from library.jiraoperations import JiraReporting +from library.slack_utility import SlackNotification +from library.ddb_issues import Operations as IssueOperations +from library.ddb_issues import IssueStatus, RedshiftEncryptionIssue +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account +from library.utility import confirm +from library.utility import SingletonInstance, SingletonInstanceException + + +class CleanRedshiftClusterUnencryption: + """ Class to remediate Redshift cluster un-encryption issues """ + def __init__(self, config): + self.config = config + + def cleanredshiftclusterunencryption(self, batch=False): + """ Class method to clean Redshift cluster which are violating aws best practices """ + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(self.config.redshiftEncrypt.ddb_table_name) + + retention_period = self.config.redshiftEncrypt.remediation_retention_period + + jira = JiraReporting(self.config, module='redshiftUnencrypted') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.aws.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_open_issues(ddb_table, account_id, RedshiftEncryptionIssue) + for issue in issues: + cluster_id = issue.issue_id + + in_whitelist = self.config.redshiftEncrypt.in_whitelist(account_id, cluster_id) + in_temp_whitelist = self.config.redshiftEncrypt.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + + if in_whitelist: + logging.debug(f"Skipping {cluster_id} (in whitelist)") + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + labels=IssueStatus.Whitelisted.value + ) + continue + + if issue.timestamps.reported is None: + logging.debug(f"Skipping '{cluster_id}' (was not reported)") + continue + + if issue.timestamps.remediated is not None: + logging.debug(f"Skipping {cluster_id} (has been already remediated)") + continue + + updated_date = issue.timestamp_as_datetime + no_of_days_issue_created = (self.config.now - updated_date).days + + if no_of_days_issue_created >= retention_period: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + try: + if not batch and \ + not confirm(f"Do you want to remediate '{cluster_id}' Redshift cluster un-encryption", False): + continue + + account = Account(id=account_id, + name=account_name, + region=issue.issue_details.region, + role_name=self.config.aws.role_name_reporting) + if account.session is None: + continue + + checker = RedshiftClusterChecker(account=account) + checker.check(clusters=[cluster_id]) + cluster_details = checker.get_cluster(cluster_id) + + if cluster_id is None: + logging.debug(f"Redshift Cluster {cluster_details.name} was removed by user") + elif cluster_details.is_encrypt: + logging.debug(f"Cluster {cluster_details.name} Un-encryption issue was remediated by user") + else: + logging.debug(f"Remediating '{cluster_details.name}' Un-encryption") + + remediation_succeed = True + if cluster_details.encrypt_cluster(): + comment = (f"Cluster '{cluster_details.name}' un-encryption issue " + f"in '{account_name} / {account_id}' account, '{issue.issue_details.region}'" + f" region was remediated by hammer") + else: + remediation_succeed = False + comment = (f"Failed to remediate cluster '{cluster_details.name}' un-encryption issue " + f"in '{account_name}/{account_id}' account, '{issue.issue_details.region}'" + f" region due to some limitations. Please, check manually") + + jira.remediate_issue( + ticket_id=issue.jira_details.ticket, + comment=comment, + reassign=remediation_succeed, + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_remediated(ddb_table, issue) + except Exception: + logging.exception(f"Error occurred while updating cluster '{cluster_id}' un-encryption " + f"in '{account_name} / {account_id}'") + else: + logging.debug(f"Skipping '{cluster_id}' " + f"({retention_period - no_of_days_issue_created} days before remediation)") + + +if __name__ == "__main__": + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + parser = argparse.ArgumentParser() + parser.add_argument('--batch', action='store_true', help='Do not ask confirmation for remediation') + args = parser.parse_args() + + try: + class_object = CleanRedshiftClusterUnencryption(config) + class_object.cleanredshiftclusterunencryption(batch=args.batch) + except Exception: + logging.exception("Failed to clean Redshift cluster unencryption") diff --git a/hammer/reporting-remediation/remediation/clean_redshift_public_access.py b/hammer/reporting-remediation/remediation/clean_redshift_public_access.py new file mode 100644 index 00000000..a2247ae3 --- /dev/null +++ b/hammer/reporting-remediation/remediation/clean_redshift_public_access.py @@ -0,0 +1,153 @@ +""" +Class to remediate Redshift cluster public access issues. +""" +import sys +import logging +import argparse + + +from library.logger import set_logging, add_cw_logging +from library.config import Config +from library.jiraoperations import JiraReporting +from library.slack_utility import SlackNotification +from library.ddb_issues import Operations as IssueOperations +from library.ddb_issues import IssueStatus, RedshiftPublicAccessIssue +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account +from library.utility import confirm +from library.utility import SingletonInstance, SingletonInstanceException + + +class CleanRedshiftPublicAccess: + """ Class to remediate Redshift cluster public access issues """ + def __init__(self, config): + self.config = config + + def clean_redshift_public_access(self, batch=False): + """ Class method to clean Redshift cluster which are violating aws best practices """ + main_account = Account(region=config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(self.config.redshift_public_access.ddb_table_name) + + retention_period = self.config.redshift_public_access.remediation_retention_period + + jira = JiraReporting(self.config, module='redshiftPublicAccess') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.aws.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_open_issues(ddb_table, account_id, RedshiftPublicAccessIssue) + for issue in issues: + cluster_id = issue.issue_id + + in_whitelist = self.config.redshift_public_access.in_whitelist(account_id, cluster_id) + in_temp_whitelist = self.config.redshift_public_access.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue + + if in_whitelist: + logging.debug(f"Skipping {cluster_id} (in whitelist)") + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + continue + + if issue.timestamps.reported is None: + logging.debug(f"Skipping '{cluster_id}' (was not reported)") + continue + + if issue.timestamps.remediated is not None: + logging.debug(f"Skipping {cluster_id} (has been already remediated)") + continue + + updated_date = issue.timestamp_as_datetime + no_of_days_issue_created = (self.config.now - updated_date).days + + if no_of_days_issue_created >= retention_period: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + try: + if not batch and \ + not confirm(f"Do you want to remediate '{cluster_id}' Redshift cluster public access", False): + continue + + account = Account(id=account_id, + name=account_name, + region= issue.issue_details.region, + role_name=self.config.aws.role_name_reporting) + if account.session is None: + continue + + checker = RedshiftClusterChecker(account=account) + checker.check(clusters=[cluster_id]) + cluster_details = checker.get_cluster(cluster_id) + + if cluster_id is None: + logging.debug(f"Redshift Cluster {cluster_details.name} was removed by user") + elif not cluster_details.is_public: + logging.debug(f"Cluster {cluster_details.name} public access issue was remediated by user") + else: + logging.debug(f"Remediating '{cluster_details.name}' public access") + + remediation_succeed = True + if cluster_details.make_private(): + comment = (f"Cluster '{cluster_details.name}' public access issue " + f"in '{account_name} / {account_id}' account, '{issue.issue_details.region}' region " + f"was remediated by hammer") + else: + remediation_succeed = False + comment = (f"Failed to remediate cluster '{cluster_details.name}' public access issue " + f"in '{account_name} / {account_id}' account, '{issue.issue_details.region}' region " + f"due to some limitations. Please, check manually") + + jira.remediate_issue( + ticket_id=issue.jira_details.ticket, + comment=comment, + reassign=remediation_succeed, + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_remediated(ddb_table, issue) + except Exception: + logging.exception(f"Error occurred while updating cluster '{cluster_id}' public access " + f"in '{account_name} / {account_id}'") + else: + logging.debug(f"Skipping '{cluster_id}' " + f"({retention_period - no_of_days_issue_created} days before remediation)") + + +if __name__ == "__main__": + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + parser = argparse.ArgumentParser() + parser.add_argument('--batch', action='store_true', help='Do not ask confirmation for remediation') + args = parser.parse_args() + + try: + class_object = CleanRedshiftPublicAccess(config) + class_object.clean_redshift_public_access(batch=args.batch) + except Exception: + logging.exception("Failed to clean Redshift cluster public access") diff --git a/hammer/reporting-remediation/remediation/clean_s3bucket_acl_permissions.py b/hammer/reporting-remediation/remediation/clean_s3bucket_acl_permissions.py index f98773e9..3d528240 100755 --- a/hammer/reporting-remediation/remediation/clean_s3bucket_acl_permissions.py +++ b/hammer/reporting-remediation/remediation/clean_s3bucket_acl_permissions.py @@ -31,7 +31,7 @@ def cleans3bucketaclpermissions(self, batch=False): retention_period = self.config.s3acl.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='s3acl') slack = SlackNotification(self.config) for account_id, account_name in self.config.s3acl.remediation_accounts.items(): @@ -42,6 +42,12 @@ def cleans3bucketaclpermissions(self, batch=False): in_whitelist = self.config.s3acl.in_whitelist(account_id, bucket_name) in_fixlist = True #self.config.s3acl.in_fixnow(account_id, bucket_name) + in_temp_whitelist = self.config.s3acl.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue if in_whitelist: logging.debug(f"Skipping {bucket_name} (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_s3bucket_policy_permissions.py b/hammer/reporting-remediation/remediation/clean_s3bucket_policy_permissions.py index 37dca5ba..4cd9823f 100755 --- a/hammer/reporting-remediation/remediation/clean_s3bucket_policy_permissions.py +++ b/hammer/reporting-remediation/remediation/clean_s3bucket_policy_permissions.py @@ -31,7 +31,7 @@ def clean_s3bucket_policy_permissions(self, batch=False): retention_period = self.config.s3policy.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='s3policy') slack = SlackNotification(self.config) for account_id, account_name in self.config.s3policy.remediation_accounts.items(): @@ -41,7 +41,13 @@ def clean_s3bucket_policy_permissions(self, batch=False): bucket_name = issue.issue_id in_whitelist = self.config.s3policy.in_whitelist(account_id, bucket_name) - #in_fixlist = self.config.s3policy.in_fixnow(account_id, bucket_name) + # in_fixlist = self.config.s3policy.in_fixnow(account_id, bucket_name) + in_temp_whitelist = self.config.s3policy.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue if in_whitelist: logging.debug(f"Skipping {bucket_name} (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_s3bucket_unencrypted.py b/hammer/reporting-remediation/remediation/clean_s3bucket_unencrypted.py index 65203d30..7a403600 100644 --- a/hammer/reporting-remediation/remediation/clean_s3bucket_unencrypted.py +++ b/hammer/reporting-remediation/remediation/clean_s3bucket_unencrypted.py @@ -30,10 +30,10 @@ def cleans3bucketunencrypted(self, batch=False): retention_period = self.config.s3Encrypt.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='s3Encrypt') slack = SlackNotification(self.config) - for account_id, account_name in self.config.aws.accounts.items(): + for account_id, account_name in self.config.s3Encrypt.remediation_accounts.items(): logging.debug(f"Checking '{account_name} / {account_id}'") issues = IssueOperations.get_account_open_issues(ddb_table, account_id, S3EncryptionIssue) for issue in issues: @@ -41,6 +41,12 @@ def cleans3bucketunencrypted(self, batch=False): in_whitelist = self.config.s3Encrypt.in_whitelist(account_id, bucket_name) in_fixlist = True + in_temp_whitelist = self.config.s3Encrypt.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue if in_whitelist: logging.debug(f"Skipping {bucket_name} (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_security_groups.py b/hammer/reporting-remediation/remediation/clean_security_groups.py index 91d0c7ad..ed3db34d 100755 --- a/hammer/reporting-remediation/remediation/clean_security_groups.py +++ b/hammer/reporting-remediation/remediation/clean_security_groups.py @@ -31,7 +31,7 @@ def clean_security_groups(self, batch=False): retention_period = self.config.sg.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='sg') slack = SlackNotification(self.config) for account_id, account_name in self.config.sg.remediation_accounts.items(): @@ -47,6 +47,14 @@ def clean_security_groups(self, batch=False): name_in_whitelist = self.config.sg.in_whitelist(account_id, f"{group_vpc_id}:{group_name}") id_in_whitelist = self.config.sg.in_whitelist(account_id, group_id) + name_in_temp_whitelist = self.config.sg.in_temp_whitelist(account_id, f"{group_vpc_id}:{group_name}") + id_in_temp_whitelist = self.config.sg.in_temp_whitelist(account_id, group_id) + if name_in_temp_whitelist or id_in_temp_whitelist: + logging.debug( + f"Skipping '{group_name}' / '{group_id}' (in temporary whitelist items." + f" Will remediate this issue in future)") + continue + if name_in_whitelist or id_in_whitelist: logging.debug(f"Skipping '{group_name} / {group_id}' (in whitelist)") diff --git a/hammer/reporting-remediation/remediation/clean_sqs_policy_permissions.py b/hammer/reporting-remediation/remediation/clean_sqs_policy_permissions.py index f5e2d0a4..d42c08d8 100644 --- a/hammer/reporting-remediation/remediation/clean_sqs_policy_permissions.py +++ b/hammer/reporting-remediation/remediation/clean_sqs_policy_permissions.py @@ -29,10 +29,10 @@ def clean_sqs_policy_permissions(self): retention_period = self.config.sqspolicy.remediation_retention_period - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='sqspolicy') slack = SlackNotification(self.config) - for account_id, account_name in self.config.aws.accounts.items(): + for account_id, account_name in self.config.sqspolicy.remediation_accounts.items(): logging.debug(f"Checking '{account_name} / {account_id}'") issues = IssueOperations.get_account_open_issues(ddb_table, account_id, SQSPolicyIssue) for issue in issues: @@ -41,6 +41,12 @@ def clean_sqs_policy_permissions(self): queue_region = issue.issue_details.region in_whitelist = self.config.sqspolicy.in_whitelist(account_id, queue_url) + in_temp_whitelist = self.config.sqspolicy.in_temp_whitelist(account_id, issue.issue_id) + if in_temp_whitelist: + logging.debug( + f"Skipping '{issue.issue_id}' (in temporary whitelist items. " + f"Will remediate this issue in future)") + continue if in_whitelist: logging.debug(f"Skipping {queue_name} (in whitelist)") diff --git a/hammer/reporting-remediation/reporting/create_cloudtrail_tickets.py b/hammer/reporting-remediation/reporting/create_cloudtrail_tickets.py index 9a9768d0..1b8dc143 100755 --- a/hammer/reporting-remediation/reporting/create_cloudtrail_tickets.py +++ b/hammer/reporting-remediation/reporting/create_cloudtrail_tickets.py @@ -46,7 +46,7 @@ def create_tickets_cloud_trail_logging(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='cloudtrails') slack = SlackNotification(self.config) for account_id, account_name in self.config.cloudtrails.accounts.items(): @@ -54,9 +54,29 @@ def create_tickets_cloud_trail_logging(self): issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, CloudTrailIssue) for issue in issues: region = issue.issue_id + + in_temp_whitelist = self.config.cloudtrails.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) \ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"CloudTrail logging issue with '{region}' " + f"is added to temporary whitelist. ") + + comment = (f"CloudTrail logging issue with '{region}' " + f"in '{account_name} / {account_id}' account is added to temporary whitelist.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + account_id=account_id + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} '{region}' CloudTrail logging issue") comment = (f"Closing {issue.status.value} issue with '{region}' CloudTrail logging in " @@ -133,7 +153,7 @@ def create_tickets_cloud_trail_logging(self): # try: # response = jira.add_issue( # issue_summary=issue_summary, issue_description=issue_description, - # priority="Major", labels=["cloud-trail-disabled"], + # priority="Major", # account_id=account_id, # ) # except Exception: diff --git a/hammer/reporting-remediation/reporting/create_ebs_public_snapshot_issue_tickets.py b/hammer/reporting-remediation/reporting/create_ebs_public_snapshot_issue_tickets.py index 204fc4f5..9609073d 100755 --- a/hammer/reporting-remediation/reporting/create_ebs_public_snapshot_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_ebs_public_snapshot_issue_tickets.py @@ -26,7 +26,7 @@ def create_tickets_ebs_public_snapshots(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='ebsSnapshot') slack = SlackNotification(self.config) for account_id, account_name in self.config.ebsSnapshot.accounts.items(): @@ -37,13 +37,33 @@ def create_tickets_ebs_public_snapshots(self): volume_id = issue.issue_details.volume_id region = issue.issue_details.region tags = issue.issue_details.tags + in_temp_whitelist = self.config.ebsSnapshot.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.jira_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"EBS public snapshot '{snapshot_id}' is added to temporary whitelist items. ") + + comment = (f"EBS public snapshot '{snapshot_id}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} EBS public snapshot '{snapshot_id}' issue") comment = (f"Closing {issue.status.value} EBS public snapshot '{snapshot_id}' issue " @@ -101,8 +121,9 @@ def create_tickets_ebs_public_snapshots(self): f"*Volume ID*: {volume_id}\n" f"\n") - auto_remediation_date = (self.config.now + self.config.ebsSnapshot.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.ebsSnapshot.remediation and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.ebsSnapshot.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -120,7 +141,7 @@ def create_tickets_ebs_public_snapshots(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["public_snapshots"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_ebs_volume_issue_tickets.py b/hammer/reporting-remediation/reporting/create_ebs_volume_issue_tickets.py index b8dc8db7..2aa4b2b6 100755 --- a/hammer/reporting-remediation/reporting/create_ebs_volume_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_ebs_volume_issue_tickets.py @@ -74,7 +74,7 @@ def create_tickets_ebsvolumes(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='ebsVolume') slack = SlackNotification(self.config) for account_id, account_name in self.config.ebsVolume.accounts.items(): @@ -84,13 +84,34 @@ def create_tickets_ebsvolumes(self): volume_id = issue.issue_id region = issue.issue_details.region tags = issue.issue_details.tags + + in_temp_whitelist = self.config.ebsVolume.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.jira_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"EBS unencrypted volume '{volume_id}' is added to temporary whitelist items. ") + + comment = (f"EBS unencrypted volume '{volume_id}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} EBS unencrypted volume '{volume_id}' issue") comment = (f"Closing {issue.status.value} EBS unencrypted volume '{volume_id}' issue " @@ -174,7 +195,8 @@ def create_tickets_ebsvolumes(self): issue_description += "*Recommendation*: Encrypt EBS volume. " if self.config.whitelisting_procedure_url: - issue_description += (f"For any other exceptions, please follow the [whitelisting procedure|{self.config.whitelisting_procedure_url}] " + issue_description += (f"For any other exceptions, please follow the " + f"[whitelisting procedure|{self.config.whitelisting_procedure_url}] " f"and provide a strong business reasoning. ") issue_summary = (f"EBS unencrypted volume '{volume_id}' " @@ -183,7 +205,7 @@ def create_tickets_ebsvolumes(self): # try: # response = jira.add_issue( # issue_summary=issue_summary, issue_description=issue_description, - # priority="Major", labels=["unencrypted-ebs-volumes"], + # priority="Major", # owner=owner, # account_id=account_id, # bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_ecs_external_image_source_issue_tickets.py b/hammer/reporting-remediation/reporting/create_ecs_external_image_source_issue_tickets.py new file mode 100644 index 00000000..8418630b --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_ecs_external_image_source_issue_tickets.py @@ -0,0 +1,182 @@ +""" +Class to create ecs external image source issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, ECSExternalImageSourceIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateECSExternalImageSourceIssueTickets(object): + """ Class to create ECS external image source issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_ecs_external_images(self): + """ Class method to create jira tickets """ + table_name = self.config.ecs_external_image_source.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='ecsExternalImageSource') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.ecs_external_image_source.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, ECSExternalImageSourceIssue) + for issue in issues: + task_definition_name = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + container_image_details = issue.issue_details.container_image_details + + in_temp_whitelist = self.config.ecs_external_image_source.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"ECS external image source '{task_definition_name}' " + f"is added to temporary whitelist items. ") + + comment = (f"ECS external image source '{task_definition_name}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} ECS external image source '{task_definition_name}' issue") + + comment = (f"Closing {issue.status.value} ECS external image source '{task_definition_name}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{task_definition_name}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting ECS external image source issue for '{task_definition_name}'") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_summary = (f"ECS external image source '{task_definition_name}'" + f"in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description = ( + f"The ECS image source taken from external source.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*ECS Task Definition*: {task_definition_name}\n" + f"*ECS container image Source*: External \n" + f"*ECS container image details*: {container_image_details} \n" + ) + + issue_description += JiraOperations.build_tags_table(tags) + + issue_description += "\n" + issue_description += ( + f"*Recommendation*: " + f"For both security and reliability, use ECS container registry and maintain " + f"all required container images within ECS. " + f"To update ECS container image source follow below steps:\n" + f"1. Open the Amazon ECS console at https://console.aws.amazon.com/ecs/. \n" + f"2. From the navigation bar, choose region that contains your task definition " + f"and choose Task Definitions.\n" + f"3. On the Task Definitions page, select the box to left of task definition to revise " + f"and choose Create new revision.\n" + f"4. On the Create new revision of Task Definition page, select the container and " + f"add internal image source to 'Image' option and then choose Update.\n" + f"5. Verify the information and choose Create.\n" + ) + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateECSExternalImageSourceIssueTickets(config) + obj.create_tickets_ecs_external_images() + except Exception: + logging.exception("Failed to create ECS external image issue tickets") diff --git a/hammer/reporting-remediation/reporting/create_ecs_logging_issue_tickets.py b/hammer/reporting-remediation/reporting/create_ecs_logging_issue_tickets.py new file mode 100644 index 00000000..43f54109 --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_ecs_logging_issue_tickets.py @@ -0,0 +1,181 @@ +""" +Class to create ecs task definition logging issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, ECSLoggingIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateECSLoggingIssueTickets(object): + """ Class to create ECS task definition logging issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_ecs_logging(self): + """ Class method to create jira tickets """ + table_name = self.config.ecs_logging.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='ecsLogging') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.ecs_logging.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, ECSLoggingIssue) + for issue in issues: + task_definition_name = issue.issue_id + disabled_logging_container_names = issue.issue_details.disabled_logging_container_names + region = issue.issue_details.region + tags = issue.issue_details.tags + + in_temp_whitelist = self.config.ecs_logging.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"ECS logging issue '{task_definition_name}' is added to temporary whitelist items. ") + + comment = (f"ECS logging issue '{task_definition_name}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} ECS logging enabled '{task_definition_name}' issue") + + comment = (f"Closing {issue.status.value} ECS logging enabled '{task_definition_name}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{task_definition_name}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting ECS logging '{task_definition_name}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_summary = (f"ECS logging is not enabled for '{task_definition_name}'" + f"in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description = ( + f"The ECS Container's logging is not enabled.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*ECS Task Definition*: {task_definition_name}\n" + f"*ECS Task definition disabled logging container names*: {disabled_logging_container_names}\n" + f"*Container's logging enabled*: False \n" + ) + + issue_description += JiraOperations.build_tags_table(tags) + + issue_description += "\n" + issue_description += ( + f"*Recommendation*: " + f"Enable logging for ECS task definition's container. To enable logging, follow below steps: \n" + f"1. Open the Amazon ECS console at https://console.aws.amazon.com/ecs/. \n" + f"2. From the navigation bar, " + f"choose region that contains your task definition and choose Task Definitions.\n" + f"3. On the Task Definitions page, select the box to the left of the task definition to revise " + f"and choose Create new revision.\n" + f"4. On the Create new revision of Task Definition page, " + f"select the container and enable 'LogConfiguration' under section 'Storage and Logging' " + f"and then choose Update.\n" + f"5. Verify the information and choose Create.\n" + ) + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateECSLoggingIssueTickets(config) + obj.create_tickets_ecs_logging() + except Exception: + logging.exception("Failed to create ecs logging tickets") diff --git a/hammer/reporting-remediation/reporting/create_ecs_privileged_access_issue_tickets.py b/hammer/reporting-remediation/reporting/create_ecs_privileged_access_issue_tickets.py new file mode 100644 index 00000000..f26cec94 --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_ecs_privileged_access_issue_tickets.py @@ -0,0 +1,185 @@ +""" +Class to create ecs privileged access issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, ECSPrivilegedAccessIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateECSPrivilegedAccessIssueTickets(object): + """ Class to create ECS privileged access issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_ecs_privileged(self): + """ Class method to create jira tickets """ + table_name = self.config.ecs_privileged_access.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='ecsPrivilegedAccess') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.ecs_privileged_access.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, ECSPrivilegedAccessIssue) + for issue in issues: + task_definition_name = issue.issue_id + privileged_container_names = issue.issue_details.privileged_container_names + region = issue.issue_details.region + tags = issue.issue_details.tags + + in_temp_whitelist = self.config.ecs_privileged_access.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"ECS privileged access issue '{task_definition_name}' " + f"is added to temporary whitelist items.") + + comment = (f"ECS privileged access issue '{task_definition_name}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} ECS privileged access disabled " + f"'{task_definition_name}' issue") + + comment = (f"Closing {issue.status.value} ECS privileged access disabled " + f"'{task_definition_name}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + labels=IssueStatus.Whitelisted + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{task_definition_name}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting ECS privileged access issue for '{task_definition_name}'") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_summary = (f"ECS privileged access is enabled for '{task_definition_name}'" + f"in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description = ( + f"The ECS privileged access is enabled.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*ECS Task Definition Name*: {task_definition_name}\n" + f"*ECS Task definition's privileged container names*: {privileged_container_names}\n" + f"*Container has privileged access*: True \n" + ) + + issue_description += JiraOperations.build_tags_table(tags) + + issue_description += "\n" + issue_description += ( + f"*Recommendation*: " + f"By default, containers are unprivileged and cannot. To disable ECS privileged access, " + f"follow below steps:" + f"1. Open the Amazon ECS console at https://console.aws.amazon.com/ecs/. \n" + f"2. From the navigation bar, " + f"choose region that contains your task definition and choose Task Definitions.\n" + f"3. On the Task Definitions page, select the box to the left of the task definition to revise " + f"and choose Create new revision.\n" + f"4. On the Create new revision of Task Definition page, " + f"select the container and disable 'Privileged' option under section 'Security' " + f"and then choose Update.\n" + f"5. Verify the information and choose Create.\n" + ) + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateECSPrivilegedAccessIssueTickets(config) + obj.create_tickets_ecs_privileged() + except Exception: + logging.exception("Failed to create ECS privileged access tickets") diff --git a/hammer/reporting-remediation/reporting/create_elasticsearch_domain_logging_issue_tickets.py b/hammer/reporting-remediation/reporting/create_elasticsearch_domain_logging_issue_tickets.py new file mode 100644 index 00000000..e8d10eee --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_elasticsearch_domain_logging_issue_tickets.py @@ -0,0 +1,177 @@ +""" +Class to create Elasticsearch domain logging issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, ESLoggingIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateElasticSearchDomainLoggingIssueTickets(object): + """ Class to create elasticsearch domain logging issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_elasticsearch_domain_logging(self): + """ Class method to create jira tickets """ + table_name = self.config.esLogging.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='esDomainLogging') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.esLogging.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, ESLoggingIssue) + for issue in issues: + domain_name = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + + in_temp_whitelist = self.config.esLogging.in_temp_whitelist(account_id, issue.issue_id) + + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"Elasticsearch logging issue '{domain_name}' is added to temporary whitelist items.") + + comment = (f"Elasticsearch domain logging issue '{domain_name}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} Elasticsearch domain logging " + f"'{domain_name}' issue") + + comment = (f"Closing {issue.status.value} Elasticsearch domain logging '{domain_name}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{domain_name}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting Elasticsearch domain logging '{domain_name}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_description = ( + f"Elasticsearch domain logging is not enabled.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*Domain ID*: {domain_name}\n" + ) + + issue_description += JiraOperations.build_tags_table(tags) + + if self.config.esLogging.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.esLogging.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + + issue_description += ( + f"*Recommendation*: " + f"Enable logging for Elasticesearch domain. " + f"Elasticsearch logs contain information about errors and warnings raised " + f"during the operation of the service and can be useful for troubleshooting.. " + ) + + issue_summary = (f"Elasticsearch domain logging issue '{domain_name}' " + f" in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateElasticSearchDomainLoggingIssueTickets(config) + obj.create_tickets_elasticsearch_domain_logging() + except Exception: + logging.exception("Failed to create Elasticsearch domain logging issue tickets") diff --git a/hammer/reporting-remediation/reporting/create_elasticsearch_public_access_issue_tickets.py b/hammer/reporting-remediation/reporting/create_elasticsearch_public_access_issue_tickets.py new file mode 100644 index 00000000..413576a5 --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_elasticsearch_public_access_issue_tickets.py @@ -0,0 +1,187 @@ +""" +Class to create Elasticsearch publicly accessible domain issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, ESPublicAccessIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateElasticSearchPublicAccessDomainTickets(object): + """ Class to create elasticsearch publicly accessible issue tickets """ + def __init__(self, config): + self.config = config + + def attachment_name(self, account_id, domain_name): + return f"{account_id}_{domain_name}_{self.config.now.isoformat('T', 'seconds')}.json" + + def create_tickets_elasticsearch_public_access(self): + """ Class method to create jira tickets """ + table_name = self.config.esPublicAccess.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='esPublicAccessDomain') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.esPublicAccess.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, ESPublicAccessIssue) + for issue in issues: + domain_name = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + policy = issue.issue_details.policy + + in_temp_whitelist = self.config.esPublicAccess.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) \ + and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"Elasticsearch publicly accessible domain issue '{domain_name}' " + f"is added to temporary whitelist items.") + + comment = (f"Elasticsearch publicly accessible domain issue '{domain_name}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} Elasticsearch publicly accessible domain '" + f"{domain_name}' issue") + + comment = (f"Closing {issue.status.value} Elasticsearch publicly accessible domain '" + f"{domain_name}' issue in '{account_name} / {account_id}' account,'{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{domain_name}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting Elasticsearch publicly accessible domain '{domain_name}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_description = ( + f"Elasticsearch domain allows unrestricted public access.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*Domain ID*: {domain_name}\n" + ) + + issue_description += JiraOperations.build_tags_table(tags) + + if self.config.esPublicAccess.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.esPublicAccess.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}" \ + f"{{color}}\n\n" + + issue_description += ( + f"*Recommendation*: " + f"Deny public access to domain. Or" + f"use AWS console which provides preconfigured access policies that can customize for specific" + f" needs of your domain. You also can import access policies from other Amazon ES domains. \n" + ) + + issue_summary = (f"Elasticsearch publicly accessible domain '{domain_name}' " + f" in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + # Adding domain policy json as attachment to Jira ticket. + jira.add_attachment(ticket_id=issue.jira_details.ticket, + filename=self.attachment_name(account_id, domain_name), + text=policy) + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateElasticSearchPublicAccessDomainTickets(config) + obj.create_tickets_elasticsearch_public_access() + except Exception: + logging.exception("Failed to create Elasticsearch publicly accessible domain tickets") diff --git a/hammer/reporting-remediation/reporting/create_elasticsearch_unencrypted_issue_tickets.py b/hammer/reporting-remediation/reporting/create_elasticsearch_unencrypted_issue_tickets.py new file mode 100644 index 00000000..3ce25abe --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_elasticsearch_unencrypted_issue_tickets.py @@ -0,0 +1,197 @@ +""" +Class to create Elasticsearch unencrypted domain issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, ESEncryptionIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateElasticSearchUnencryptedDomainTickets(object): + """ Class to create elasticsearch unencryption issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_elasticsearch_unencryption(self): + """ Class method to create jira tickets """ + table_name = self.config.esEncrypt.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='esUnencryptedDomain') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.esEncrypt.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, ESEncryptionIssue) + for issue in issues: + domain_name = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + encrypted_at_rest = issue.issue_details.encrypted_at_rest + encrypted_at_transit = issue.issue_details.encrypted_at_transit + + in_temp_whitelist = self.config.esEncrypt.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"Elasticsearch unencrypted domain issue '{domain_name}' " + f"is added to temporary whitelist items.") + + comment = (f"Elasticsearch unencrypted domain issue '{domain_name}' " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} Elasticsearch unencrypted domain " + f"'{domain_name}' issue") + + comment = (f"Closing {issue.status.value} Elasticsearch unencrypted domain " + f"'{domain_name}' issue in '{account_name} / {account_id}' account, '{region}' " + f"region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{domain_name}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting Elasticsearch unencrypted domain '{domain_name}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_description = "" + + if not encrypted_at_rest and not encrypted_at_transit: + issue_description += ( + f"Elasticsearch domain needs to be encrypt at rest and transit. \n\n" + ) + issue_summary = (f"Elasticsearch unencrypted domain '{domain_name}' " + f" in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + elif not encrypted_at_transit: + issue_description += ( + f"Elasticsearch domain needs to be encrypt at transit. \n\n" + ) + issue_summary = (f"Elasticsearch domain '{domain_name}' unencrypted at transit" + f" in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + elif not encrypted_at_rest: + issue_description += ( + f"Elasticsearch domain needs to encrypted at rest. \n\n" + ) + issue_summary = (f"Elasticsearch unencrypted domain '{domain_name}' unencrypted at rest" + f" in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description += ( + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*Domain ID*: {domain_name}\n" + f"*Encryption enabled at rest*: {encrypted_at_rest}\n" + f"*Encryption enabled in transit*: {encrypted_at_transit}\n" + ) + + issue_description += JiraOperations.build_tags_table(tags) + + issue_description += ( + f"*Recommendation*: Encrypt Elasticsearch domain. To enable encryption follow below steps: \n" + f"1. Choose to create new domain. \n" + f"2. Enable both node-node encryption and encryption at rest options.\n" + f"3. Fill other domain configuration details and navigate to review page. \n" + f"4. On the Review page, review your domain configuration, and then choose 'Confirm' to " + f"create new domain. \n " + f"5. After creation of new domain, migrate your data to new domain. \n " + ) + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateElasticSearchUnencryptedDomainTickets(config) + obj.create_tickets_elasticsearch_unencryption() + except Exception: + logging.exception("Failed to create Elasticsearch unencrypted domain tickets") diff --git a/hammer/reporting-remediation/reporting/create_iam_key_inactive_tickets.py b/hammer/reporting-remediation/reporting/create_iam_key_inactive_tickets.py index f43b7555..a835ba2a 100755 --- a/hammer/reporting-remediation/reporting/create_iam_key_inactive_tickets.py +++ b/hammer/reporting-remediation/reporting/create_iam_key_inactive_tickets.py @@ -27,7 +27,7 @@ def create_jira_ticket(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='iamUserInactiveKeys') slack = SlackNotification(self.config) for account_id, account_name in self.config.iamUserInactiveKeys.accounts.items(): @@ -36,9 +36,29 @@ def create_jira_ticket(self): for issue in issues: key_id = issue.issue_id username = issue.issue_details.username + + in_temp_whitelist = self.config.iamUserInactiveKeys.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"IAM Inactive access key issue '{key_id} / {username}' is " + f"added to temporary whitelist items.") + + comment = (f"IAM Inactive access key issue '{key_id} / {username}' " + f"in '{account_name} / {account_id}' account is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + account_id=account_id + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} inactive access key '{key_id} / {username}' issue") comment = (f"Closing {issue.status.value} inactive access key '{key_id} / {username}' issue " @@ -82,8 +102,10 @@ def create_jira_ticket(self): f"*Key last used*: {last_used}\n" f"\n") - auto_remediation_date = (self.config.now + self.config.iamUserInactiveKeys.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.iamUserInactiveKeys.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.iamUserInactiveKeys.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += f"*Recommendation*: Deactivate specified inactive user access key. " @@ -94,7 +116,7 @@ def create_jira_ticket(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["inactive-iam-keys"], + priority="Major", account_id=account_id, ) except Exception: diff --git a/hammer/reporting-remediation/reporting/create_iam_key_rotation_tickets.py b/hammer/reporting-remediation/reporting/create_iam_key_rotation_tickets.py index 74fd5872..0c3dde43 100755 --- a/hammer/reporting-remediation/reporting/create_iam_key_rotation_tickets.py +++ b/hammer/reporting-remediation/reporting/create_iam_key_rotation_tickets.py @@ -27,7 +27,7 @@ def create_jira_ticket(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='iamUserKeysRotation') slack = SlackNotification(self.config) for account_id, account_name in self.config.iamUserKeysRotation.accounts.items(): @@ -36,9 +36,29 @@ def create_jira_ticket(self): for issue in issues: key_id = issue.issue_id username = issue.issue_details.username + + in_temp_whitelist = self.config.iamUserKeysRotation.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug( + f"IAM stale access key issue '{key_id} / {username}' " + f"is added to temporary whitelist items.") + + comment = (f"IAM stale access key issue '{key_id} / {username}' " + f"in '{account_name} / {account_id}' account is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + account_id=account_id + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing stale access key {issue.status.value} '{key_id} / {username}' issue") comment = (f"Closing {issue.status.value} stale access key '{key_id} / {username}' issue " @@ -92,7 +112,7 @@ def create_jira_ticket(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["iam-key-rotation"], + priority="Major", account_id=account_id, ) except Exception: diff --git a/hammer/reporting-remediation/reporting/create_public_ami_issue_tickets.py b/hammer/reporting-remediation/reporting/create_public_ami_issue_tickets.py index 3cf6b7f5..eb599de4 100644 --- a/hammer/reporting-remediation/reporting/create_public_ami_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_public_ami_issue_tickets.py @@ -26,7 +26,7 @@ def create_tickets_public_ami(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='publicAMIs') slack = SlackNotification(self.config) for account_id, account_name in self.config.publicAMIs.accounts.items(): @@ -36,13 +36,35 @@ def create_tickets_public_ami(self): ami_id = issue.issue_id ami_region = issue.issue_details.region tags = issue.issue_details.tags + + in_temp_whitelist = self.config.publicAMIs.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.issue_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) \ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"AMI '{ami_id}' is added to temporary whitelist items.") + + comment = (f"AMI '{ami_id}' public access issue " + f"in '{account_name} / {account_id}' account, {ami_region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} AMI '{ami_id}' public access issue") comment = (f"Closing {issue.status.value} AMI '{ami_id}' public access issue " @@ -112,8 +134,10 @@ def create_tickets_public_ami(self): f"*AMI Id*: {ami_id}\n" f"\n") - auto_remediation_date = (self.config.now + self.config.publicAMIs.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.publicAMIs.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.publicAMIs.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -126,7 +150,7 @@ def create_tickets_public_ami(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["public-ami"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_rds_public_snapshot_issue_tickets.py b/hammer/reporting-remediation/reporting/create_rds_public_snapshot_issue_tickets.py index e0227dca..a6052346 100755 --- a/hammer/reporting-remediation/reporting/create_rds_public_snapshot_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_rds_public_snapshot_issue_tickets.py @@ -26,7 +26,7 @@ def create_tickets_rds_public_snapshots(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='rdsSnapshot') slack = SlackNotification(self.config) for account_id, account_name in self.config.rdsSnapshot.accounts.items(): @@ -36,13 +36,35 @@ def create_tickets_rds_public_snapshots(self): snapshot_id = issue.issue_id region = issue.issue_details.region tags = issue.issue_details.tags + + in_temp_whitelist = self.config.rdsSnapshot.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.jira_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) \ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"RDS public snapshot '{snapshot_id}' is added to temporary whitelist items.") + + comment = (f"RDS public snapshot '{snapshot_id}' issue " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} RDS public snapshot '{snapshot_id}' issue") comment = (f"Closing {issue.status.value} RDS public snapshot '{snapshot_id}' issue " @@ -98,8 +120,10 @@ def create_tickets_rds_public_snapshots(self): f"*Region*: {region}\n" f"*RDS Snapshot ID*: {snapshot_id}\n") - auto_remediation_date = (self.config.now + self.config.rdsSnapshot.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.rdsSnapshot.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.rdsSnapshot.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -117,7 +141,7 @@ def create_tickets_rds_public_snapshots(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["rds-public-snapshots"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_rds_unencrypted_instance_issue_tickets.py b/hammer/reporting-remediation/reporting/create_rds_unencrypted_instance_issue_tickets.py index 10a71429..a38f50c3 100644 --- a/hammer/reporting-remediation/reporting/create_rds_unencrypted_instance_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_rds_unencrypted_instance_issue_tickets.py @@ -26,7 +26,7 @@ def create_tickets_rds_unencrypted_instances(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='rdsEncrypt') slack = SlackNotification(self.config) for account_id, account_name in self.config.aws.accounts.items(): @@ -37,13 +37,36 @@ def create_tickets_rds_unencrypted_instances(self): instance_name = issue.issue_details.name region = issue.issue_details.region tags = issue.issue_details.tags + + in_temp_whitelist = self.config.rdsEncrypt.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.jira_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist])\ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"RDS unencrypted instance '{instance_name}' " + f"is added to temporary whitelist items.") + + comment = (f"RDS unencrypted instance '{instance_name}' issue " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} RDS unencrypted instance '{instance_name}' issue") comment = (f"Closing {issue.status.value} RDS unencrypted instance '{instance_name}' issue " @@ -114,7 +137,7 @@ def create_tickets_rds_unencrypted_instances(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["rds-unencrypted-instances"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_redshift_logging_issue_tickets.py b/hammer/reporting-remediation/reporting/create_redshift_logging_issue_tickets.py new file mode 100644 index 00000000..82ff0588 --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_redshift_logging_issue_tickets.py @@ -0,0 +1,178 @@ +""" +Class to create redshift cluster logging issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, RedshiftLoggingIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateRedshiftLoggingIssueTickets(object): + """ Class to create redshift cluster logging issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_redshift_logging(self): + """ Class method to create jira tickets """ + table_name = self.config.redshift_logging.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='redshiftLogging') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.aws.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, RedshiftLoggingIssue) + for issue in issues: + cluster_id = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + + in_temp_whitelist = self.config.redshift_logging.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"Redshift cluster logging '{cluster_id}' is added to temporary whitelist items.") + + comment = (f"Redshift cluster logging '{cluster_id}' issue " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} Redshift logging '{cluster_id}' issue") + + comment = (f"Closing {issue.status.value} Redshift cluster logging '{cluster_id}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{cluster_id}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting Redshift cluster logging '{cluster_id}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_summary = (f"Redshift logging is not enabled for '{cluster_id}'" + f"in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description = ( + f"The Redshift Cluster audit logging is not enabled.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*Redshift Cluster ID*: {cluster_id}\n") + + issue_description += JiraOperations.build_tags_table(tags) + + issue_description += "\n" + issue_description += ( + f"*Recommendation*: " + f"Enable logging for Redshift cluster. To enable logging, follow below steps:\n\n" + f"1. Sign in to the AWS Management Console and open the Amazon Redshift console.\n" + f"2. In the navigation pane, click Clusters.\n" + f"3. In the list, click the cluster for which you want to enable logging.\n" + f"4. In the cluster details page, click Database, and then click Configure Audit Logging.\n" + f"5. In the Configure Audit Logging dialog box, in the Enable Audit Logging box, click Yes.\n" + f"6. For S3 Bucket, do one of the following:\n" + f" (a)If you already have an S3 bucket that you want to use, " + f"select Use Existing and then select the bucket from the Bucket list.\n" + f" (b)If you need a new S3 bucket,select Create New, and in New Bucket Name box, type a name.\n" + f"7. Optionally, in the S3 Key Prefix box, type a prefix to add to the S3 bucket.\n" + f"8. Click Save \n\n" + ) + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateRedshiftLoggingIssueTickets(config) + obj.create_tickets_redshift_logging() + except Exception: + logging.exception("Failed to create redshift cluster logging tickets") diff --git a/hammer/reporting-remediation/reporting/create_redshift_public_access_issue_tickets.py b/hammer/reporting-remediation/reporting/create_redshift_public_access_issue_tickets.py new file mode 100644 index 00000000..13141036 --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_redshift_public_access_issue_tickets.py @@ -0,0 +1,174 @@ +""" +Class to create redshift publicly accessible cluster issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, RedshiftPublicAccessIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateRedshiftPublicAccessTickets(object): + """ Class to create redshift publicly accessible cluster issue issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_redshift_public_access(self): + """ Class method to create jira tickets """ + table_name = self.config.redshift_public_access.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='redshiftPublicAccess') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.aws.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, RedshiftPublicAccessIssue) + for issue in issues: + cluster_id = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + + in_temp_whitelist = self.config.redshift_public_access.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"Redshift publicly accessible cluster issue '{cluster_id}' " + f"is added to temporary whitelist items.") + + comment = (f"Redshift publicly accessible cluster '{cluster_id}' issue " + f"in '{account_name} / {account_id}' account, {region} " + f"region added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} Redshift publicly accessible " + f"cluster '{cluster_id}' issue") + + comment = (f"Closing {issue.status.value} Redshift publicly accessible cluster " + f"'{cluster_id}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{cluster_id}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting Redshift publicly accessible cluster '{cluster_id}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_summary = (f"Redshift publicly accessible cluster '{cluster_id}'" + f"in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description = ( + f"The Redshift Cluster is publicly accessible.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*Redshift Cluster ID*: {cluster_id}\n") + + if self.config.redshift_public_access.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.redshift_public_access.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + + issue_description += JiraOperations.build_tags_table(tags) + + issue_description += "\n" + issue_description += ( + f"*Recommendation*: " + f"Disable public access of the Redshift cluster.") + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateRedshiftPublicAccessTickets(config) + obj.create_tickets_redshift_public_access() + except Exception: + logging.exception("Failed to create redshift publicly accessible cluster tickets") diff --git a/hammer/reporting-remediation/reporting/create_redshift_unencrypted_cluster_issue_tickets.py b/hammer/reporting-remediation/reporting/create_redshift_unencrypted_cluster_issue_tickets.py new file mode 100644 index 00000000..5abeeae1 --- /dev/null +++ b/hammer/reporting-remediation/reporting/create_redshift_unencrypted_cluster_issue_tickets.py @@ -0,0 +1,174 @@ +""" +Class to create redshift unencrypted cluster issue tickets. +""" +import sys +import logging + + +from library.logger import set_logging, add_cw_logging +from library.aws.utility import Account +from library.config import Config +from library.jiraoperations import JiraReporting, JiraOperations +from library.slack_utility import SlackNotification +from library.ddb_issues import IssueStatus, RedshiftEncryptionIssue +from library.ddb_issues import Operations as IssueOperations +from library.utility import SingletonInstance, SingletonInstanceException + + +class CreateRedshiftUnencryptedInstanceTickets(object): + """ Class to create redshift unencrypted cluster issue issue tickets """ + def __init__(self, config): + self.config = config + + def create_tickets_redshift_unencrypted_cluster(self): + """ Class method to create jira tickets """ + table_name = self.config.redshiftEncrypt.ddb_table_name + + main_account = Account(region=self.config.aws.region) + ddb_table = main_account.resource("dynamodb").Table(table_name) + jira = JiraReporting(self.config, module='redshiftUnencrypted') + slack = SlackNotification(self.config) + + for account_id, account_name in self.config.aws.accounts.items(): + logging.debug(f"Checking '{account_name} / {account_id}'") + issues = IssueOperations.get_account_not_closed_issues(ddb_table, account_id, RedshiftEncryptionIssue) + for issue in issues: + cluster_id = issue.issue_id + region = issue.issue_details.region + tags = issue.issue_details.tags + + in_temp_whitelist = self.config.redshiftEncrypt.in_temp_whitelist(account_id, issue.issue_id) + # issue has been already reported + if issue.timestamps.reported is not None: + owner = issue.jira_details.owner + bu = issue.jira_details.business_unit + product = issue.jira_details.product + + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist])\ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"Redshift unencrypted cluster issue '{cluster_id}' " + f"is added to temporary whitelist items.") + + comment = (f"Redshift unencrypted cluster '{cluster_id}' issue " + f"in '{account_name} / {account_id}' account, {region} " + f"region is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + logging.debug(f"Closing {issue.status.value} Redshift unencrypted cluster '{cluster_id}' issue") + + comment = (f"Closing {issue.status.value} Redshift unencrypted cluster '{cluster_id}' issue " + f"in '{account_name} / {account_id}' account, '{region}' region") + if issue.status == IssueStatus.Whitelisted: + # Adding label with "whitelisted" to jira ticket. + jira.add_label( + ticket_id=issue.jira_details.ticket, + label=IssueStatus.Whitelisted.value + ) + jira.close_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_closed(ddb_table, issue) + else: + logging.debug(f"No changes for '{cluster_id}'") + # issue has not been reported yet + else: + logging.debug(f"Reporting Redshift unencrypted cluster '{cluster_id}' issue") + + owner = tags.get("owner", None) + bu = tags.get("bu", None) + product = tags.get("product", None) + + issue_summary = (f"Redshift unencrypted cluster '{cluster_id}'" + f"in '{account_name} / {account_id}' account{' [' + bu + ']' if bu else ''}") + + issue_description = ( + f"The Redshift Cluster is unencrypted.\n\n" + f"*Risk*: High\n\n" + f"*Account Name*: {account_name}\n" + f"*Account ID*: {account_id}\n" + f"*Region*: {region}\n" + f"*Redshift Cluster ID*: {cluster_id}\n") + + issue_description += JiraOperations.build_tags_table(tags) + + if self.config.redshiftEncrypt.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.redshiftEncrypt.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + + issue_description += "\n" + issue_description += ( + f"*Recommendation*: \n" + f"Modify an unencrypted cluster.\n\n" + ) + + try: + response = jira.add_issue( + issue_summary=issue_summary, issue_description=issue_description, + priority="Major", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + except Exception: + logging.exception("Failed to create jira ticket") + continue + + if response is not None: + issue.jira_details.ticket = response.ticket_id + issue.jira_details.ticket_assignee_id = response.ticket_assignee_id + + issue.jira_details.owner = owner + issue.jira_details.business_unit = bu + issue.jira_details.product = product + + slack.report_issue( + msg=f"Discovered {issue_summary}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + + IssueOperations.set_status_reported(ddb_table, issue) + + +if __name__ == '__main__': + module_name = sys.modules[__name__].__loader__.name + set_logging(level=logging.DEBUG, logfile=f"/var/log/hammer/{module_name}.log") + config = Config() + add_cw_logging(config.local.log_group, + log_stream=module_name, + level=logging.DEBUG, + region=config.aws.region) + try: + si = SingletonInstance(module_name) + except SingletonInstanceException: + logging.error(f"Another instance of '{module_name}' is already running, quitting") + sys.exit(1) + + try: + obj = CreateRedshiftUnencryptedInstanceTickets(config) + obj.create_tickets_redshift_unencrypted_cluster() + except Exception: + logging.exception("Failed to create redshift unencrypted cluster tickets") diff --git a/hammer/reporting-remediation/reporting/create_s3_unencrypted_bucket_issue_tickets.py b/hammer/reporting-remediation/reporting/create_s3_unencrypted_bucket_issue_tickets.py index f8b2fdb5..d6d43666 100644 --- a/hammer/reporting-remediation/reporting/create_s3_unencrypted_bucket_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_s3_unencrypted_bucket_issue_tickets.py @@ -26,7 +26,7 @@ def create_tickets_s3_unencrypted_buckets(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='s3Encrypt') slack = SlackNotification(self.config) for account_id, account_name in self.config.aws.accounts.items(): @@ -35,13 +35,34 @@ def create_tickets_s3_unencrypted_buckets(self): for issue in issues: bucket_name = issue.issue_id tags = issue.issue_details.tags + + in_temp_whitelist = self.config.s3Encrypt.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.issue_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"S3 bucket unencrypted issue '{bucket_name}' " + f"is added to temporary whitelist items.") + + comment = (f"S3 bucket unencrypted '{bucket_name}' issue " + f"in '{account_name} / {account_id}' account is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} S3 bucket '{bucket_name}' unencrypted issue") comment = (f"Closing {issue.status.value} S3 bucket '{bucket_name}' unencrypted issue " @@ -113,8 +134,10 @@ def create_tickets_s3_unencrypted_buckets(self): f"*Bucket Owner*: {owner}\n" f"\n") - auto_remediation_date = (self.config.now + self.config.s3Encrypt.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.s3Encrypt.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.s3Encrypt.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -127,7 +150,7 @@ def create_tickets_s3_unencrypted_buckets(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["s3-unencrypted"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_s3bucket_acl_issue_tickets.py b/hammer/reporting-remediation/reporting/create_s3bucket_acl_issue_tickets.py index 8fad3747..4a010def 100755 --- a/hammer/reporting-remediation/reporting/create_s3bucket_acl_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_s3bucket_acl_issue_tickets.py @@ -33,7 +33,7 @@ def create_tickets_s3buckets(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='s3acl') slack = SlackNotification(self.config) for account_id, account_name in self.config.s3acl.accounts.items(): @@ -42,13 +42,34 @@ def create_tickets_s3buckets(self): for issue in issues: bucket_name = issue.issue_id tags = issue.issue_details.tags + + in_temp_whitelist = self.config.s3acl.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.issue_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"S3 bucket public ACL issue '{bucket_name}' " + f"is added to temporary whitelist items.") + + comment = (f"S3 bucket public ACL '{bucket_name}' issue " + f"in '{account_name} / {account_id}' account is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} S3 bucket '{bucket_name}' public ACL issue") comment = (f"Closing {issue.status.value} S3 bucket '{bucket_name}' public ACL issue " @@ -118,8 +139,10 @@ def create_tickets_s3buckets(self): f"*Bucket Owner*: {owner}\n" f"\n") - auto_remediation_date = (self.config.now + self.config.s3acl.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.s3acl.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.s3acl.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -139,7 +162,7 @@ def create_tickets_s3buckets(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["publics3"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_s3bucket_policy_issue_tickets.py b/hammer/reporting-remediation/reporting/create_s3bucket_policy_issue_tickets.py index b4411daa..1e0657fa 100755 --- a/hammer/reporting-remediation/reporting/create_s3bucket_policy_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_s3bucket_policy_issue_tickets.py @@ -29,7 +29,7 @@ def create_tickets_s3buckets(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='s3policy') slack = SlackNotification(self.config) for account_id, account_name in self.config.s3policy.accounts.items(): @@ -39,13 +39,34 @@ def create_tickets_s3buckets(self): bucket_name = issue.issue_id tags = issue.issue_details.tags policy = issue.issue_details.policy + + in_temp_whitelist = self.config.s3policy.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.issue_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (issue.status in [IssueStatus.Tempwhitelist] or in_temp_whitelist) and issue.timestamps.temp_whitelisted is None: + logging.debug(f"S3 bucket public policy issue '{bucket_name}' " + f"is added to temporary whitelist items.") + + comment = (f"S3 bucket public policy '{bucket_name}' issue " + f"in '{account_name} / {account_id}' account is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} S3 bucket '{bucket_name}' public policy issue") comment = (f"Closing {issue.status.value} S3 bucket '{bucket_name}' public policy " @@ -122,8 +143,10 @@ def create_tickets_s3buckets(self): f"*Bucket Owner*: {owner}\n" f"\n") - auto_remediation_date = (self.config.now + self.config.s3policy.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.s3policy.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.s3policy.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -142,7 +165,7 @@ def create_tickets_s3buckets(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["publics3"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_security_groups_tickets.py b/hammer/reporting-remediation/reporting/create_security_groups_tickets.py index 3ade7aff..e5bdaef7 100755 --- a/hammer/reporting-remediation/reporting/create_security_groups_tickets.py +++ b/hammer/reporting-remediation/reporting/create_security_groups_tickets.py @@ -3,11 +3,7 @@ """ import sys import logging -import warnings - -from functools import lru_cache -from ipwhois import IPWhois from collections import Counter from library.logger import set_logging, add_cw_logging from library.config import Config @@ -21,35 +17,23 @@ from library.aws.utility import Account from library.aws.security_groups import RestrictionStatus from library.aws.rds import RDSOperations +from library.aws.ecs import ECSClusterOperations +from library.aws.redshift import RedshiftClusterOperations +from library.aws.elasticsearch import ElasticSearchOperations from library.utility import SingletonInstance, SingletonInstanceException +from library.utility import get_registrant class CreateSecurityGroupsTickets(object): """ Class to create jira tickets for security group issues """ + def __init__(self, config): self.config = config @staticmethod - @lru_cache(maxsize=128) def get_registrant(cidr): - ip = cidr.split("/")[0] - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - try: - whois = IPWhois(ip).lookup_rdap() - except Exception: - return "" - - registrants = [] - for title, obj in whois.get('objects', {}).items(): - if obj.get('contact') is None: - continue - if 'registrant' in obj.get('roles', []): - registrants.append(f"{obj['contact'].get('name')} ({title})") - break - - return ', '.join(registrants) + registrant = get_registrant(cidr) + return f"{registrant['name']} ({registrant['title']})" if registrant else "" def build_open_ports_table_jira(self, perms): open_partly = any([perm['status'] == 'open_partly' for perm in perms]) @@ -123,7 +107,7 @@ def build_instances_table(self, iam_client, instances): product = ec2_instance.tags.get('product') component = ec2_instance.tags.get('component') if self.config.jira.text_field_character_limit == 0 or \ - len(instance_details) < (self.config.jira.text_field_character_limit * 0.5): + len(instance_details) < (self.config.jira.text_field_character_limit * 0.5): instance_details += ( f"|{ec2_instance.id}|{ec2_instance.state}" f"|{list_converter(ec2_instance.private_ips)}" @@ -138,7 +122,8 @@ def build_instances_table(self, iam_client, instances): instance_profile_id = ec2_instance.iam_profile_id if instance_profile_id is not None: try: - public_role_policies = IAMOperations.get_instance_profile_policy_details(iam_client, instance_profile_id) + public_role_policies = IAMOperations.get_instance_profile_policy_details(iam_client, + instance_profile_id) except Exception: logging.exception("Failed to get instance profile policy details") public_role_policies = [] @@ -170,9 +155,9 @@ def build_instances_table(self, iam_client, instances): if len(instance_profile_details) > 0: instance_profile_details = ( - f"\n*Instance Role Unsafe Policies:*\n" - f"||Instance Id||Role Name||Policy Name||Unsafe actions||\n" - ) + "".join(instance_profile_details) + "\n" + f"\n*Instance Role Unsafe Policies:*\n" + f"||Instance Id||Role Name||Policy Name||Unsafe actions||\n" + ) + "".join(instance_profile_details) + "\n" return instance_details, instance_profile_details, in_use, public, blind_public, owner, bu, product @@ -214,13 +199,64 @@ def build_elb_instances_table(elb_details): return elb_instance_details, in_use + @staticmethod + def build_ecs_clusters_table(ecs_clusters): + cluster_details = "" + in_use = False + + if len(ecs_clusters) > 0: + in_use = True + cluster_details += ( + f"\n*ECS Clustes:*\n" + f"||ECS Cluster ID||ECS Instance ARN||\n") + for cluster in ecs_clusters: + cluster_details += ( + f"|{cluster.cluster_arn}|{cluster.cluster_instance_arn}|\n" + ) + + return cluster_details, in_use + + @staticmethod + def build_redshift_clusters_table(redshift_clusters): + cluster_details = "" + in_use = False + + if len(redshift_clusters) > 0: + in_use = True + cluster_details += ( + f"\n*Redshift Clustes:*\n" + f"||Redshift Cluster ID||Subnet_Group_Name||\n") + for cluster in redshift_clusters: + cluster_details += ( + f"|{cluster.id}|{cluster.subnet_group_name}|\n" + ) + + return cluster_details, in_use + + @staticmethod + def build_es_domains_table(es_domains): + domain_details = "" + in_use = False + + if len(es_domains) > 0: + in_use = True + domain_details += ( + f"\n*Elasticsearch Domains:*\n" + f"||Domain Name||Domain Arn||\n") + for domain in es_domains: + domain_details += ( + f"|{domain.domain_name}|{domain.domain_arn}|\n" + ) + + return domain_details, in_use + def create_tickets_securitygroups(self): """ Class function to create jira tickets """ table_name = self.config.sg.ddb_table_name main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='sg') slack = SlackNotification(self.config) for account_id, account_name in self.config.sg.accounts.items(): @@ -232,13 +268,36 @@ def create_tickets_securitygroups(self): group_region = issue.issue_details.region group_vpc_id = issue.issue_details.vpc_id tags = issue.issue_details.tags + + in_temp_whitelist = self.config.sg.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.jira_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) \ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"Insecure security group issue '{group_name} / {group_id}' " + f"is added to temporary whitelist items.") + + comment = (f"Insecure security group '{group_name} / {group_id}' issue " + f"in '{account_name} / {account_id}' account, {group_region} " + f"region is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} security group '{group_name} / {group_id}' issue") comment = (f"Closing {issue.status.value} security group '{group_name} / {group_id}' issue " @@ -313,7 +372,7 @@ def create_tickets_securitygroups(self): ec2_client = account.client("ec2") if account.session is not None else None sg_instance_details = ec2_owner = ec2_bu = ec2_product = None - sg_in_use = sg_in_use_ec2 = sg_in_use_elb = sg_in_use_rds = None + sg_in_use = sg_in_use_ec2 = sg_in_use_elb = sg_in_use_rds = sg_in_use_ecs = sg_in_use_redshift = sg_in_use_es = None sg_public = sg_blind_public = False rds_client = account.client("rds") if account.session is not None else None @@ -322,29 +381,67 @@ def create_tickets_securitygroups(self): iam_client = account.client("iam") if account.session is not None else None - rds_instance_details = elb_instance_details = None + ecs_client = account.client("ecs") if account.session is not None else None + rds_instance_details = elb_instance_details = sg_redshift_details = sg_ecs_details = sg_es_details = None + redshift_client = account.client("redshift") if account.session is not None else None + + es_client = account.client("es") if account.session is not None else None if ec2_client is not None: ec2_instances = EC2Operations.get_instance_details_of_sg_associated(ec2_client, group_id) - sg_instance_details, instance_profile_details,\ - sg_in_use_ec2, sg_public, sg_blind_public, \ - ec2_owner, ec2_bu, ec2_product = self.build_instances_table(iam_client, ec2_instances) + sg_instance_details, instance_profile_details, \ + sg_in_use_ec2, sg_public, sg_blind_public, \ + ec2_owner, ec2_bu, ec2_product = self.build_instances_table(iam_client, ec2_instances) if elb_client is not None and elbv2_client is not None: try: - elb_instances = EC2Operations.get_elb_details_of_sg_associated(elb_client, elbv2_client, group_id) + elb_instances = EC2Operations.get_elb_details_of_sg_associated(elb_client, elbv2_client, + group_id) elb_instance_details, sg_in_use_elb = self.build_elb_instances_table(elb_instances) except Exception: - logging.exception(f"Failed to build ELB details for '{group_name} / {group_id}' in {account}") + logging.exception( + f"Failed to build ELB details for '{group_name} / {group_id}' in {account}") if rds_client is not None: try: - rds_instances = RDSOperations.get_rds_instance_details_of_sg_associated(rds_client, group_id) + rds_instances = RDSOperations.get_rds_instance_details_of_sg_associated(rds_client, + group_id) rds_instance_details, sg_in_use_rds = self.build_rds_instances_table(rds_instances) except Exception: - logging.exception(f"Failed to build RDS details for '{group_name} / {group_id}' in {account}") + logging.exception( + f"Failed to build RDS details for '{group_name} / {group_id}' in {account}") - sg_in_use = sg_in_use_ec2 or sg_in_use_elb or sg_in_use_rds + if ecs_client is not None: + try: + ecs_instances = ECSClusterOperations.get_ecs_instance_security_groups(ec2_client, + ecs_client, group_id) + sg_ecs_details, sg_in_use_ecs = self.build_ecs_clusters_table(ecs_instances) + + except Exception: + logging.exception( + f"Failed to build ECS Cluster details for '{group_name} / {group_id}' in {account}") + + if redshift_client is not None: + try: + redshift_clusters = RedshiftClusterOperations.get_redshift_vpc_security_groups( + redshift_client, group_id) + sg_redshift_details, sg_in_use_redshift = self.build_redshift_clusters_table( + redshift_clusters) + except Exception: + logging.exception( + f"Failed to build Redshift Cluster details for '{group_name} / {group_id}' in {account}") + + if es_client is not None: + try: + es_domains = ElasticSearchOperations.get_elasticsearch_details_of_sg_associated( + es_client, group_id) + sg_es_details, sg_in_use_es = self.build_es_domains_table( + es_domains) + except Exception: + logging.exception( + f"Failed to build Redshift Cluster details for '{group_name} / {group_id}' in {account}") + + sg_in_use = sg_in_use_ec2 or sg_in_use_elb or sg_in_use_rds or sg_in_use_redshift or sg_in_use_ecs or sg_in_use_es owner = group_owner if group_owner is not None else ec2_owner bu = group_bu if group_bu is not None else ec2_bu @@ -424,7 +521,8 @@ def create_tickets_securitygroups(self): f"{threat}" f"{account_details}") - if status == RestrictionStatus.OpenCompletely: + if (status == RestrictionStatus.OpenCompletely) \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): auto_remediation_date = (self.config.now + self.config.sg.issue_retention_date).date() issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" @@ -438,14 +536,21 @@ def create_tickets_securitygroups(self): issue_description += f"{instance_profile_details if instance_profile_details else ''}" + issue_description += f"{sg_ecs_details if sg_ecs_details else ''}" + + issue_description += f"{sg_redshift_details if sg_redshift_details else ''}" + + issue_description += f"{sg_es_details if sg_es_details else ''}" + issue_description += ( f"*Recommendation*: " f"Allow access only for a minimum set of required ip addresses/ranges from [RFC1918|https://tools.ietf.org/html/rfc1918]. " ) if self.config.whitelisting_procedure_url: - issue_description += (f"For any other exceptions, please follow the [whitelisting procedure|{self.config.whitelisting_procedure_url}] " - f"and provide a strong business reasoning. ") + issue_description += ( + f"For any other exceptions, please follow the [whitelisting procedure|{self.config.whitelisting_procedure_url}] " + f"and provide a strong business reasoning. ") issue_description += f"Be sure to delete overly permissive rules after creating rules that are more restrictive.\n" @@ -455,7 +560,7 @@ def create_tickets_securitygroups(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority=priority, labels=["insecure-services"], + priority=priority, owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/hammer/reporting-remediation/reporting/create_sqs_policy_issue_tickets.py b/hammer/reporting-remediation/reporting/create_sqs_policy_issue_tickets.py index 2f452024..e5fa0fb2 100644 --- a/hammer/reporting-remediation/reporting/create_sqs_policy_issue_tickets.py +++ b/hammer/reporting-remediation/reporting/create_sqs_policy_issue_tickets.py @@ -29,7 +29,7 @@ def create_tickets_sqs_policy(self): main_account = Account(region=self.config.aws.region) ddb_table = main_account.resource("dynamodb").Table(table_name) - jira = JiraReporting(self.config) + jira = JiraReporting(self.config, module='sqspolicy') slack = SlackNotification(self.config) for account_id, account_name in self.config.aws.accounts.items(): @@ -41,13 +41,36 @@ def create_tickets_sqs_policy(self): queue_region = issue.issue_details.region tags = issue.issue_details.tags policy = issue.issue_details.policy + + in_temp_whitelist = self.config.sqspolicy.in_temp_whitelist(account_id, issue.issue_id) # issue has been already reported if issue.timestamps.reported is not None: owner = issue.issue_details.owner bu = issue.jira_details.business_unit product = issue.jira_details.product - if issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: + if (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]) \ + and issue.timestamps.temp_whitelisted is None: + logging.debug(f"SQS queue public policy issue '{queue_name}' " + f"is added to temporary whitelist items.") + + comment = (f"SQS queue public policy '{queue_name}' issue " + f"in '{account_name} / {account_id}' account, {queue_region} " + f"region is added to temporary whitelist items.") + jira.update_issue( + ticket_id=issue.jira_details.ticket, + comment=comment + ) + + slack.report_issue( + msg=f"{comment}" + f"{' (' + jira.ticket_url(issue.jira_details.ticket) + ')' if issue.jira_details.ticket else ''}", + owner=owner, + account_id=account_id, + bu=bu, product=product, + ) + IssueOperations.set_status_temp_whitelisted(ddb_table, issue) + elif issue.status in [IssueStatus.Resolved, IssueStatus.Whitelisted]: logging.debug(f"Closing {issue.status.value} SQS queue '{queue_name}' public policy issue") comment = (f"Closing {issue.status.value} SQS queue '{queue_name}' public policy " @@ -125,9 +148,10 @@ def create_tickets_sqs_policy(self): f"*SQS queue name*: {queue_name}\n" f"*SQS queue region*: {queue_region}\n" f"\n") - - auto_remediation_date = (self.config.now + self.config.sqspolicy.issue_retention_date).date() - issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" + if self.config.sqspolicy.remediation \ + and not (in_temp_whitelist or issue.status in [IssueStatus.Tempwhitelist]): + auto_remediation_date = (self.config.now + self.config.sqspolicy.issue_retention_date).date() + issue_description += f"\n{{color:red}}*Auto-Remediation Date*: {auto_remediation_date}{{color}}\n\n" issue_description += JiraOperations.build_tags_table(tags) @@ -146,7 +170,7 @@ def create_tickets_sqs_policy(self): try: response = jira.add_issue( issue_summary=issue_summary, issue_description=issue_description, - priority="Major", labels=["publicsqs"], + priority="Major", owner=owner, account_id=account_id, bu=bu, product=product, diff --git a/requirements.txt b/requirements.txt index 59f3ec29..25089c94 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pyflakes -boto3==1.7.84 +boto3==1.9.42 moto==1.3.5 watchtower pytest diff --git a/tests/mock_ecs.py b/tests/mock_ecs.py new file mode 100644 index 00000000..4e60ebf1 --- /dev/null +++ b/tests/mock_ecs.py @@ -0,0 +1,45 @@ +import boto3 +import logging + +from moto import mock_ecs +from library.utility import jsonDumps + + +def start(): + """ + Entrypoint for mocking ecs. + :return: nothing + """ + # start ECS mocking with moto + mock = mock_ecs() + mock.start() + + +def create_env_task_definitions(task_definitions, region): + logging.debug(f"======> creating new ECS task definitions from {jsonDumps(task_definitions)}") + ecs_client = boto3.client("ecs", region_name=region) + + test_task_definitions = [] + + for task_definition, rule in task_definitions.items(): + ecs_client.register_task_definition( + family=task_definition, + containerDefinitions=rule["containerDefinitions"] + ) + + test_task_definitions.append(task_definition) + + # remove moto precreated task definitions + task_definitions_list_to_check = ecs_client.list_task_definition_families() + for task_definition in task_definitions_list_to_check["families"]: + if task_definition not in test_task_definitions: + ecs_client.deregister_task_definition( + taskDefinition=task_definition + ) + + task_definitions = ecs_client.list_task_definition_families()["families"] + logging.debug(f"{jsonDumps(task_definitions)}") + + # need to return task definitions + return test_task_definitions + diff --git a/tests/mock_redshift.py b/tests/mock_redshift.py new file mode 100644 index 00000000..11aec911 --- /dev/null +++ b/tests/mock_redshift.py @@ -0,0 +1,55 @@ +import boto3 +import logging + +from moto import mock_redshift +from library.utility import jsonDumps + + +def start(): + """ + Entrypoint for mocking ecs. + :return: nothing + """ + # start ECS mocking with moto + mock = mock_redshift() + mock.start() + + +def create_env_clusters(clusters, region): + logging.debug(f"======> creating new Redshift clusters from {jsonDumps(clusters)}") + redshift_client = boto3.client("redshift", region_name=region) + + test_clusters = [] + clusters_list = [] + + for cluster, rule in clusters.items(): + cluster_id = redshift_client.create_cluster( + DBName=rule["DBName"], + ClusterIdentifier=cluster, + ClusterType=rule["ClusterType"], + NodeType=rule["NodeType"], + MasterUsername=rule["MasterUsername"], + MasterUserPassword=rule["MasterUserPassword"], + PubliclyAccessible=rule["PubliclyAccessible"], + Encrypted=rule["Encrypted"] + )["Cluster"]["ClusterIdentifier"] + + test_clusters.append(cluster_id) + + # remove moto precreated clusters + redshift_clusters_list_to_check = redshift_client.describe_clusters() + for cluster in redshift_clusters_list_to_check["Clusters"]: + + if cluster["ClusterIdentifier"] not in test_clusters: + redshift_client.delete_cluster( + ClusterIdentifier=cluster["ClusterIdentifier"], + SkipFinalClusterSnapshot=True + ) + else: + clusters_list.append(cluster["ClusterIdentifier"]) + + logging.debug(f"{jsonDumps(clusters_list)}") + + # need to return task definitions + return test_clusters + diff --git a/tests/test_ecs_external_image_source.py b/tests/test_ecs_external_image_source.py new file mode 100644 index 00000000..c1af0864 --- /dev/null +++ b/tests/test_ecs_external_image_source.py @@ -0,0 +1,84 @@ +import pytest + +from . import mock_ecs +from library.aws.ecs import ECSChecker +from library.aws.utility import Account + +region = "us-east-1" + +task_definitions = { + "tas_definition": { + "family": 'test_ecs_image_source', + "Description": "Congainer image taken from external source", + "CheckShouldPass": False, + "containerDefinitions": [ + { + 'name': 'hello_world1', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True + }, + { + 'name': 'hello_world2', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True + } + ] + } +} + + +def find_task_definition_name(task_definition_details): + for taskDefinition, props in task_definitions.items(): + if taskDefinition == task_definition_details.name: + return taskDefinition + return None + + +def ident_task_definition_test(task_definition_details): + """ + Used to build identification string for each autogenerated test (for easy recognition of failed tests). + + :param task_definition_details: dict with information about rules from + ECSChecker(...) + :return: identification string with task_definition_name. + """ + + name = find_task_definition_name(task_definition_details) + descr = task_definitions.get(name, {}).get("Description", "default description") + return f"params: {name} ({descr})" + + +def pytest_generate_tests(metafunc): + """ + Entrypoint for tests (built-in pytest function for dynamic generation of test cases). + """ + # Launch ECS mocking and env preparation + mock_ecs.start() + test_task_definitions = mock_ecs.create_env_task_definitions(task_definitions, region) + + account = Account(region=region) + + # validate ebs volumes in mocked env + checker = ECSChecker(account) + checker.check(task_definitions=test_task_definitions) + + # create test cases for each response + metafunc.parametrize("task_definition_details", checker.task_definitions, ids=ident_task_definition_test) + + +@pytest.mark.ecs_external_image_source +def test_task(task_definition_details): + """ + Actual testing function. + + :param task_definition_details: dict with information about rules from + ECSChecker(...) + :return: nothing, raises AssertionError if actual test result is not matched with expected + """ + name = find_task_definition_name(task_definition_details) + expected = task_definitions.get(name, {})["CheckShouldPass"] + assert expected == (not task_definition_details.external_image) \ No newline at end of file diff --git a/tests/test_ecs_logging.py b/tests/test_ecs_logging.py new file mode 100644 index 00000000..0f8bfcaa --- /dev/null +++ b/tests/test_ecs_logging.py @@ -0,0 +1,106 @@ +import pytest + +from . import mock_ecs +from library.aws.ecs import ECSChecker +from library.aws.utility import Account + +region = "us-east-1" + +task_definitions = { + "tas_definition1": { + "Description": "ECS task definition's logging is enabled.", + "CheckShouldPass": True, + "containerDefinitions": [ + { + 'name': 'hello_world1', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'logConfiguration': {'logDriver': 'json-file'} + }, + { + 'name': 'hello_world2', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'logConfiguration': {'logDriver': 'json-file'} + } + ] + }, + "tas_definition2": { + "Description": "ECS task definition's logging is not enabled.", + "CheckShouldPass": False, + "containerDefinitions": [ + { + 'name': 'hello_world3', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True + }, + { + 'name': 'hello_world4', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True + } + ] + } + +} + + +def find_task_definition_name(task_definition_details): + for taskDefinition, props in task_definitions.items(): + if taskDefinition == task_definition_details.name: + return taskDefinition + return None + + +def ident_task_definition_test(task_definition_details): + """ + Used to build identification string for each autogenerated test (for easy recognition of failed tests). + + :param task_definition_details: dict with information about rules from + ECSChecker(...) + :return: identification string with task_definition_name. + """ + + name = find_task_definition_name(task_definition_details) + descr = task_definitions.get(name, {}).get("Description", "default description") + return f"params: {name} ({descr})" + + +def pytest_generate_tests(metafunc): + """ + Entrypoint for tests (built-in pytest function for dynamic generation of test cases). + """ + # Launch ECS mocking and env preparation + mock_ecs.start() + test_task_definitions = mock_ecs.create_env_task_definitions(task_definitions, region) + + account = Account(region=region) + + # validate ebs volumes in mocked env + checker = ECSChecker(account) + checker.check(task_definitions=test_task_definitions) + + # create test cases for each response + metafunc.parametrize("task_definition_details", checker.task_definitions, ids=ident_task_definition_test) + + +@pytest.mark.ecs_logging +def test_task(task_definition_details): + """ + Actual testing function. + + :param task_definition_details: dict with information about rules from + ECSChecker(...) + :return: nothing, raises AssertionError if actual test result is not matched with expected + """ + name = find_task_definition_name(task_definition_details) + expected = task_definitions.get(name, {})["CheckShouldPass"] + assert expected == task_definition_details.is_logging diff --git a/tests/test_ecs_privileged_access.py b/tests/test_ecs_privileged_access.py new file mode 100644 index 00000000..0169eff8 --- /dev/null +++ b/tests/test_ecs_privileged_access.py @@ -0,0 +1,91 @@ +import pytest + +from . import mock_ecs +from library.aws.ecs import ECSChecker +from library.aws.utility import Account + +region = "us-east-1" + +task_definitions = { + "tas_definition1": { + "Description": "ECS task enabled privileged access", + "CheckShouldPass": False, + "containerDefinitions": [ + { + 'name': 'hello_world1', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'privileged': True + } + ] + }, + "tas_definition2": { + "Description": "ECS task disabled privileged access", + "CheckShouldPass": True, + "containerDefinitions": [ + { + 'name': 'hello_world2', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'privileged': False + } + ] + } +} + + +def find_task_definition_name(task_definition_details): + for taskDefinition, props in task_definitions.items(): + if taskDefinition == task_definition_details.name: + return taskDefinition + return None + + +def ident_task_definition_test(task_definition_details): + """ + Used to build identification string for each autogenerated test (for easy recognition of failed tests). + + :param task_definition_details: dict with information about rules from + ECSChecker(...) + :return: identification string with task_definition_name. + """ + + name = find_task_definition_name(task_definition_details) + descr = task_definitions.get(name, {}).get("Description", "default description") + return f"params: {name} ({descr})" + + +def pytest_generate_tests(metafunc): + """ + Entrypoint for tests (built-in pytest function for dynamic generation of test cases). + """ + # Launch ECS mocking and env preparation + mock_ecs.start() + test_task_definitions = mock_ecs.create_env_task_definitions(task_definitions, region) + + account = Account(region=region) + + # validate ebs volumes in mocked env + checker = ECSChecker(account) + checker.check(task_definitions=test_task_definitions) + + # create test cases for each response + metafunc.parametrize("task_definition_details", checker.task_definitions, ids=ident_task_definition_test) + + +@pytest.mark.ecs_privileged_access +def test_task(task_definition_details): + """ + Actual testing function. + + :param task_definition_details: dict with information about rules from + ECSChecker(...) + :return: nothing, raises AssertionError if actual test result is not matched with expected + """ + name = find_task_definition_name(task_definition_details) + expected = task_definitions.get(name, {})["CheckShouldPass"] + assert expected == (not task_definition_details.is_privileged) diff --git a/tests/test_redshift_encryption.py b/tests/test_redshift_encryption.py new file mode 100644 index 00000000..adb3908a --- /dev/null +++ b/tests/test_redshift_encryption.py @@ -0,0 +1,97 @@ +import pytest + +from . import mock_redshift +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account + +region = "us-east-1" + +clusters = { + "cluster1": { + "Description": "Encrypted Cluster", + "DBName": "test1", + "ClusterType": "single-node", + "NodeType": "ds2.xlarge", + "MasterUsername": "user1", + "MasterUserPassword": "testUser1password123", + "Encrypted": True, + "PubliclyAccessible": False, + "CheckShouldPass": True + }, + "cluster2": { + "Description": "Unencrypted Cluster", + "DBName": "test2", + "ClusterType": "single-node", + "NodeType": "ds2.xlarge", + "MasterUsername": "user2", + "MasterUserPassword": "testUser2password123", + "Encrypted": False, + "PubliclyAccessible": False, + "CheckShouldPass": False + } +} + + +def find_rule_prop(cluster_details, prop, default): + + try: + return clusters[cluster_details.name][prop] + except KeyError: + return default + + +def ident_cluster_test(arg): + """ + Used to build identification string for each autogenerated test (for easy recognition of failed tests). + + :param cluster_details: dict with information about rules from + RedshiftClusterChecker(...) + :return: identification string with cluster name. + """ + if isinstance(arg, bool): + return "remediated" if arg else "original" + else: + descr = find_rule_prop(arg, "Description", "default description") + return f"params: {arg.name} ({descr})" + + +def pytest_generate_tests(metafunc): + """ + Entrypoint for tests (built-in pytest function for dynamic generation of test cases). + """ + # Launch Redshift mocking and env preparation + mock_redshift.start() + test_clusters = mock_redshift.create_env_clusters(clusters, region) + + account = Account(region=region) + + # validate ebs volumes in mocked env + checker = RedshiftClusterChecker(account) + checker.check(clusters=test_clusters) + + for cluster in checker.clusters: + cluster.encrypt_cluster() + + checker_remediated = RedshiftClusterChecker(account) + checker_remediated.check() + + redshift_clusters = [(cluster, False) for cluster in checker.clusters] + redshift_clusters += [(cluster, True) for cluster in checker_remediated.clusters] + + # create test cases for each response + metafunc.parametrize("cluster,remediated", redshift_clusters, ids=ident_cluster_test) + + +@pytest.mark.redshift_public_access +def test_cluster(cluster, remediated): + """ + Actual testing function. + + :param cluster: dict with information about rules from + RedshiftClusterChecker(...) + :param remediated: remediation details + :return: nothing, raises AssertionError if actual test result is not matched with expected + """ + + expected = True if remediated else find_rule_prop(cluster, "CheckShouldPass", True) + assert expected == (cluster.is_encrypt) diff --git a/tests/test_redshift_logging.py b/tests/test_redshift_logging.py new file mode 100644 index 00000000..a2660476 --- /dev/null +++ b/tests/test_redshift_logging.py @@ -0,0 +1,74 @@ +import pytest + +from . import mock_redshift +from library.aws.redshift import RedshiftLoggingChecker +from library.aws.utility import Account + +region = "us-east-1" + +clusters = { + "cluster1": { + "Description": "Cluster logging disabled", + "DBName": "test1", + "ClusterType": "single-node", + "NodeType": "ds2.xlarge", + "MasterUsername": "user1", + "MasterUserPassword": "testUser1password123", + "Encrypted": True, + "PubliclyAccessible": False, + "CheckShouldPass": True + } +} + + +def find_cluster_name(cluster_details): + for cluster, props in clusters.items(): + if props["ClusterIdentifier"] == cluster_details.name: + return cluster + return None + + +def ident_cluster_test(cluster_details): + """ + Used to build identification string for each autogenerated test (for easy recognition of failed tests). + + :param cluster_details: dict with information about rules from + RedshiftClusterChecker(...) + :return: identification string with cluster name. + """ + + name = find_cluster_name(cluster_details) + descr = clusters.get(name, {}).get("Description", "default description") + return f"params: {name} ({descr})" + + +def pytest_generate_tests(metafunc): + """ + Entrypoint for tests (built-in pytest function for dynamic generation of test cases). + """ + # Launch Redshift mocking and env preparation + mock_redshift.start() + test_clusters = mock_redshift.create_env_clusters(clusters, region) + + account = Account(region=region) + + # validate ebs volumes in mocked env + checker = RedshiftLoggingChecker(account) + checker.check(clusters=test_clusters) + + # create test cases for each response + metafunc.parametrize("cluster_details", checker.clusters, ids=ident_cluster_test) + + +@pytest.mark.redshift_public_access +def test_cluster(cluster_details): + """ + Actual testing function. + + :param cluster_details: dict with information about rules from + RedshiftClusterChecker(...) + :return: nothing, raises AssertionError if actual test result is not matched with expected + """ + name = find_cluster_name(cluster_details) + expected = clusters.get(name, {})["CheckShouldPass"] + assert expected == (not cluster_details.is_logging) diff --git a/tests/test_redshift_public_access.py b/tests/test_redshift_public_access.py new file mode 100644 index 00000000..48ae7bee --- /dev/null +++ b/tests/test_redshift_public_access.py @@ -0,0 +1,97 @@ +import pytest + +from . import mock_redshift +from library.aws.redshift import RedshiftClusterChecker +from library.aws.utility import Account + +region = "us-east-1" + +clusters = { + "cluster1": { + "Description": "Cluster has public access", + "DBName": "test1", + "ClusterType": "single-node", + "NodeType": "ds2.xlarge", + "MasterUsername": "user1", + "MasterUserPassword": "testUser1password123", + "PubliclyAccessible": True, + "Encrypted": True, + "CheckShouldPass": False + }, + "cluster2": { + "Description": "Cluster has private access", + "DBName": "test2", + "ClusterType": "single-node", + "NodeType": "ds2.xlarge", + "MasterUsername": "user2", + "MasterUserPassword": "testUser2password123", + "PubliclyAccessible": False, + "Encrypted": True, + "CheckShouldPass": True + } +} + + +def find_rule_prop(cluster_details, prop, default): + + try: + return clusters[cluster_details.name][prop] + except KeyError: + return default + + +def ident_cluster_test(arg): + """ + Used to build identification string for each autogenerated test (for easy recognition of failed tests). + + :param cluster_details: dict with information about rules from + RedshiftClusterChecker(...) + :return: identification string with cluster name. + """ + if isinstance(arg, bool): + return "remediated" if arg else "original" + else: + descr = find_rule_prop(arg, "Description", "default description") + return f"params: {arg.name} ({descr})" + + +def pytest_generate_tests(metafunc): + """ + Entrypoint for tests (built-in pytest function for dynamic generation of test cases). + """ + # Launch Redshift mocking and env preparation + mock_redshift.start() + test_clusters = mock_redshift.create_env_clusters(clusters, region) + + account = Account(region=region) + + # validate ebs volumes in mocked env + checker = RedshiftClusterChecker(account) + checker.check(clusters=test_clusters) + + for cluster in checker.clusters: + if cluster.is_public: + cluster.modify_cluster(False) + + checker_remediated = RedshiftClusterChecker(account) + checker_remediated.check() + + redshift_clusters = [(cluster, False) for cluster in checker.clusters] + redshift_clusters += [(cluster, True) for cluster in checker_remediated.clusters] + + # create test cases for each response + metafunc.parametrize("cluster, remediated", redshift_clusters, ids=ident_cluster_test) + + +@pytest.mark.redshift_public_access +def test_cluster(cluster, remediated): + """ + Actual testing function. + + :param cluster_details: dict with information about rules from + RedshiftClusterChecker(...) + :param remediated: remediation details. + :return: nothing, raises AssertionError if actual test result is not matched with expected + """ + expected = True if remediated else find_rule_prop(cluster, "CheckShouldPass", True) + assert expected == (not cluster.is_public) diff --git a/tox.ini b/tox.ini index 058670e4..b076c47f 100755 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,12 @@ python_paths = hammer/identification/lambdas/ebs-unencrypted-volume-identification hammer/identification/lambdas/ebs-public-snapshots-identification hammer/identification/lambdas/sqs-public-policy-identification + hammer/identification/lambdas/redshift-cluster-public-access-identification + hammer/identification/lambdas/redshift-unencrypted-cluster-identification + hammer/identification/lambdas/redshift-audit-logging-issues-identification + hammer/identification/lambdas/ecs-privileged-access-issues-identification + hammer/identification/lambdas/ecs-logging-issues-identification + hammer/identification/lambdas/ecs-external-image-source-issues-identification hammer [flake8]