Optional snapshot copy to alternate region

ec2-automate-backup/README.md

@@ -7,12 +7,33 @@ ec2-automate-backup was created to provide easy backup/snapshot functionality fo
 
 # Installation Instructions:
 ec2-automate-backup requires the AWS Command Line Interface tool be installed and properly configured. Instructions for installing the AWS Command Line Interface tool is available at https://aws.amazon.com/cli/.
+## Policy ( optional )
+You can omit environmental configuration by giving the policy to entire EC2 instance where you run the script with the following policy:
+```json
+{
+    "Statement": [
+        {
+            "Action": [
+                "ec2:DescribeVolumes",
+                "ec2:CreateSnapshot",
+                "ec2:DescribeSnapshots",
+                "ec2:DeleteSnapshot",
+                "ec2:CreateTags",
+                "ec2:CopySnapshot",
+                "ec2:DescribeTags"
+            ],
+            "Resource": "*",
+            "Effect": "Allow"
+        }
+    ]
+}
+```
 
 # Directions For Use:
 ## Example of Use:
-`ec2-automate-backup.sh -v vol-6d6a0527`
+`./ec2-automate-backup.sh -r us-west-2 -s tag -t "Backup,Values=true" -k 44 -p -u -n -a`
 
-the above example would provide a single backup of the EBS volumeid vol-6d6a0527. The snapshot would be created with the description "vol-6d6a0527_2012-09-07".
+the above example would provide a backup of all EBS volumes from 'Oregon' with tag 'Backup' set to 'true' with additional copy to 'Ireland'.
 ## Required Parameters:
 ec2-automate-backup requires one of the following two parameters be provided:
 
@@ -36,6 +57,10 @@ ec2-automate-backup requires one of the following two parameters be provided:
 
 `-u` - the -u flag will tag snapshots with additional data so that snapshots can be more easily located. Currently the two user tags created are Volume="ebs_volume" and Created="date." These can be easily modified in code.
 
+`-a` - the -a flag will make additional copy of the snapshot to alternate AWS region ( defaults to eu-west-1 )
+
+`-g <AWS_region>` - the AWS region for additional copy of the snapshot
+
 # Potential Uses and Methods of Use:
 * To backup multiple EBS volumes use ec2-automate-backup as follows: `ec2-automate-backup.sh -v "vol-6d6a0527 vol-636a0112"`
 * To backup a selected group of EBS volumes on a daily schedule tag each volume you wish to backup with the tag "Backup=true" and run ec2-automate-backup using cron as follows: `0 0 * * * ec2-automate-backup.sh -s tag -t "Backup,Values=true"`
@@ -45,9 +70,6 @@ ec2-automate-backup requires one of the following two parameters be provided:
 * To perform daily backup using cron and to load environment configuration with a "cron-primer" file:
  - `0 0 * * * ec2-user /home/ec2-user/ec2-automate-backup.sh -c /home/ec2-user/cron-primer.sh -s tag -t "Backup,Values=true"`
 
-# Additional Information:
-the file "ec2ab - IAM User Required Permissions.json" contains the IAM permissions required to run ec2-automate-backup.sh in with the least permissions required as of 2012-11-21.
-
 - Author: Colin Johnson / [email protected]
 - Date: 2015-10-26
 - Version 0.10

ec2-automate-backup/ec2-automate-backup.sh

@@ -51,12 +51,23 @@ get_EBS_List() {
   fi
 }
 
+#get_Snapshot_List gets a list of just created snapshots
+get_Snapshot_List() {
+  #creates a list of all just created snapshots in $region
+  ebs_snapshot_list=$(aws ec2 describe-snapshots --region $region --filters Name=tag:Created,Values=$current_date --output text --query 'Snapshots[*].SnapshotId')
+  #takes the output of the previous command 
+  ebs_snapshot_list_result=$(echo $?)
+  if [[ $ebs_snapshot_list_result -gt 0 ]]; then
+    echo -e "An error occurred when running ec2-describe-snapshots. The error returned is below:\n$ebs_snapshot_list_result" 1>&2 ; exit 70
+  fi
+}
+
 create_EBS_Snapshot_Tags() {
   #snapshot tags holds all tags that need to be applied to a given snapshot - by aggregating tags we ensure that ec2-create-tags is called only onece
   snapshot_tags="Key=CreatedBy,Value=ec2-automate-backup"
   #if $name_tag_create is true then append ec2ab_${ebs_selected}_$current_date to the variable $snapshot_tags
   if $name_tag_create; then
-    snapshot_tags="$snapshot_tags Key=Name,Value=ec2ab_${ebs_selected}_$current_date"
+    snapshot_tags="$snapshot_tags Key=Name,Value=$ec2_snapshot_description"
   fi
   #if $hostname_tag_create is true then append --tag InitiatingHost=$(hostname -f) to the variable $snapshot_tags
   if $hostname_tag_create; then
@@ -78,6 +89,33 @@ create_EBS_Snapshot_Tags() {
   fi
 }
 
+create_Alternate_Snapshot_Tags() {
+  #snapshot tags holds all tags that need to be applied to a given snapshot - by aggregating tags we ensure that ec2-create-tags is called only onece
+  snapshot_tags="Key=CreatedBy,Value=ec2-automate-backup"
+  #if $name_tag_create is true then append ec2ab_${ebs_selected}_$current_date to the variable $snapshot_tags
+  if $name_tag_create; then
+    snapshot_tags="$snapshot_tags Key=Name,Value=$ec2_snapshot_description"
+  fi
+  #if $hostname_tag_create is true then append --tag InitiatingHost=$(hostname -f) to the variable $snapshot_tags
+  if $hostname_tag_create; then
+    snapshot_tags="$snapshot_tags Key=InitiatingHost,Value='$(hostname -s)'"
+  fi
+  #if $purge_after_date_fe is true, then append $purge_after_date_fe to the variable $snapshot_tags
+  if [[ -n $purge_after_date_fe ]]; then
+    snapshot_tags="$snapshot_tags Key=PurgeAfterFE,Value=$purge_after_date_fe Key=PurgeAllow,Value=true"
+  fi
+  #if $user_tags is true, then append Volume=$ebs_selected and Created=$current_date to the variable $snapshot_tags
+  if $user_tags; then
+    snapshot_tags="$snapshot_tags Key=Volume,Value=${volume_id} Key=Created,Value=$current_date"
+  fi
+  #if $snapshot_tags is not zero length then set the tag on the snapshot using aws ec2 create-tags
+  if [[ -n $snapshot_tags ]]; then
+    echo "Tagging Snapshot $alternate_snapshot_resource_id with the following Tags: $snapshot_tags"
+    tags_argument="--tags $snapshot_tags"
+    aws_ec2_create_tag_result=$(aws ec2 create-tags --resources $alternate_snapshot_resource_id --region $alternate_region $tags_argument --output text 2>&1)
+  fi
+}
+
 get_date_binary() {
   #$(uname -o) (operating system) would be ideal, but OS X / Darwin does not support to -o option
   #$(uname) on OS X defaults to $(uname -s) and $(uname) on GNU/Linux defaults to $(uname -s)
@@ -133,6 +171,30 @@ purge_EBS_Snapshots() {
   done
 }
 
+purge_Alternate_Snapshots() {
+  # snapshot_purge_allowed is a string containing the SnapshotIDs of snapshots
+  # that contain a tag with the key value/pair PurgeAllow=true
+  snapshot_purge_allowed=$(aws ec2 describe-snapshots --region $alternate_region --filters Name=tag:PurgeAllow,Values=true --output text --query 'Snapshots[*].SnapshotId')
+
+  for snapshot_id_evaluated in $snapshot_purge_allowed; do
+    #gets the "PurgeAfterFE" date which is in UTC with UNIX Time format (or xxxxxxxxxx / %s)
+    purge_after_fe=$(aws ec2 describe-snapshots --region $alternate_region --snapshot-ids $snapshot_id_evaluated --output text | grep ^TAGS.*PurgeAfterFE | cut -f 3)
+    #if purge_after_date is not set then we have a problem. Need to alert user.
+    if [[ -z $purge_after_fe ]]; then
+      #Alerts user to the fact that a Snapshot was found with PurgeAllow=true but with no PurgeAfterFE date.
+      echo "Snapshot with the Snapshot ID \"$snapshot_id_evaluated\" has the tag \"PurgeAllow=true\" but does not have a \"PurgeAfterFE=xxxxxxxxxx\" key/value pair. $app_name is unable to determine if $snapshot_id_evaluated should be purged." 1>&2
+    else
+      # if $purge_after_fe is less than $current_date then
+      # PurgeAfterFE is earlier than the current date
+      # and the snapshot can be safely purged
+      if [[ $purge_after_fe < $current_date ]]; then
+        echo "Snapshot \"$snapshot_id_evaluated\" with the PurgeAfterFE date of \"$purge_after_fe\" will be deleted."
+        aws_ec2_delete_snapshot_result=$(aws ec2 delete-snapshot --region $alternate_region --snapshot-id $snapshot_id_evaluated --output text 2>&1)
+      fi
+    fi
+  done
+}
+
 #calls prerequisitecheck function to ensure that all executables required for script execution are available
 prerequisite_check
 
@@ -149,16 +211,22 @@ hostname_tag_create=false
 user_tags=false
 #sets the Purge Snapshot feature to false - if purge_snapshots=true then snapshots will be purged
 purge_snapshots=false
-#handles options processing
+#sets the copy snapshot to alternate region to false for backward compatibility
+copy_to_alternate_region=false
+#default alternate region for snapshot copy
+alternate_region="eu-west-1"
 
-while getopts :s:c:r:v:t:k:pnhu opt; do
+#handles options processing
+while getopts :s:c:r:v:t:k:g:apnhu opt; do
   case $opt in
     s) selection_method="$OPTARG" ;;
     c) cron_primer="$OPTARG" ;;
     r) region="$OPTARG" ;;
     v) volumeid="$OPTARG" ;;
     t) tag="$OPTARG" ;;
     k) purge_after_input="$OPTARG" ;;
+    g) alternate_region="$OPTARG" ;;
+    a) copy_to_alternate_region=true ;;
     n) name_tag_create=true ;;
     h) hostname_tag_create=true ;;
     p) purge_snapshots=true ;;
@@ -204,8 +272,11 @@ get_EBS_List
 
 #the loop below is called once for each volume in $ebs_backup_list - the currently selected EBS volume is passed in as "ebs_selected"
 for ebs_selected in $ebs_backup_list; do
-  ec2_snapshot_description="ec2ab_${ebs_selected}_$current_date"
+	volume_name="Unknown"
+	volume_name=$(aws ec2 describe-tags --region $region --filters "Name=resource-id,Values=$ebs_selected" "Name=resource-type,Values=volume" "Name=key,Values=Name" --query 'Tags[0].{Value:Value}' --output text 2>&1)
+  ec2_snapshot_description="${volume_name}_${ebs_selected}_$current_date"
   ec2_snapshot_resource_id=$(aws ec2 create-snapshot --region $region --description $ec2_snapshot_description --volume-id $ebs_selected --output text --query SnapshotId 2>&1)
+  ec2_create_snapshot_result=$(echo $?)
   if [[ $? != 0 ]]; then
     echo -e "An error occurred when running ec2-create-snapshot. The error returned is below:\n$ec2_create_snapshot_result" 1>&2 ; exit 70
   fi  
@@ -217,3 +288,30 @@ if $purge_snapshots; then
   echo "Snapshot Purging is Starting Now."
   purge_EBS_Snapshots
 fi
+
+#echo "Sleeping until snapshots complete."
+#sleep 2m
+
+#the loop below is called once for each snapshot in $ebs_snapshot_list - the currently selected snapshot is passed in as "ebs_selected"
+if $copy_to_alternate_region; then
+	get_Snapshot_List
+	for ebs_selected in $ebs_snapshot_list; do
+		volume_id=$(aws ec2 describe-tags --region $region --filters "Name=resource-id,Values=$ebs_selected" "Name=resource-type,Values=snapshot" "Name=key,Values=Volume" --query 'Tags[0].{Value:Value}' --output text 2>&1)
+		volume_name="Unknown"
+		volume_name=$(aws ec2 describe-tags --region $region --filters "Name=resource-id,Values=$volume_id" "Name=resource-type,Values=volume" "Name=key,Values=Name" --query 'Tags[0].{Value:Value}' --output text 2>&1)
+		ec2_snapshot_description="${volume_name}_${volume_id}_$current_date"
+		alternate_snapshot_resource_id=$(aws --region $alternate_region ec2 copy-snapshot --source-region $region --description $ec2_snapshot_description --source-snapshot-id $ebs_selected --output text --query SnapshotId 2>&1)
+		ec2_create_snapshot_result=$(echo $?)
+		if [[ $? != 0 ]]; then
+			echo -e "An error occurred when running ec2-copy-snapshot. The error returned is below:\n$ec2_create_snapshot_result" 1>&2 ; exit 70
+		fi
+		create_Alternate_Snapshot_Tags
+		# Trying to manage the AWS limit of 5 concurrent cross-region copies. Choose something between 1-2 minutes depending on size and distance.
+		sleep 60
+	done
+fi
+
+if $purge_snapshots && $copy_to_alternate_region; then
+  echo "Alternate Snapshot Purging is Starting Now."
+  purge_Alternate_Snapshots
+fi