backup to s3

aws cli

install

python, pip and awscli if necessary

python --version
sudo apt-get install python2.7
echo 'alias python=python2.7'  >> ~/.bashrc
bash --login
sudo apt install python-pip
sudo pip install awscli
aws --version

configure

aws configure enter key and secret from aws_keys file. make a new user and login at 'https://console.aws.amazon.com/iam/home?region=us-west-2#/users' if you lose that file

backup script

save this to bin/db_backup and make it executable chmod +x bin/db_backup

#!/bin/bash

export PATH=/usr/local/bin:$PATH
bucket_name=raceweb-brainpad
backup_path=backup/current
current_date=`date +%Y-%m-%d-%H-%M-%S`

echo "##########################################################"
echo `date`
echo "backing up $current_date.zip to S3 bucket: $bucket_name..."
echo "##########################################################"

# make the backup directory unless it exists
if [ -d $backup_path ]; then
    echo "$backup_path already exists, cleaning it out..."
    rm $backup_path/*.json
    rm $backup_path/*.zip
else
    echo "making $backup_path"
    mkdir $backup_path
fi

mongoexport --db brainpad_production --collection accounts --out $backup_path/accounts.json
mongoexport --db brainpad_production --collection connections --out $backup_path/connections.json
mongoexport --db brainpad_production --collection contacts --out $backup_path/contacts.json
mongoexport --db brainpad_production --collection journals --out $backup_path/journals.json
mongoexport --db brainpad_production --collection links --out $backup_path/links.json
mongoexport --db brainpad_production --collection lookups --out $backup_path/lookups.json
mongoexport --db brainpad_production --collection milestones --out $backup_path/milestones.json
mongoexport --db brainpad_production --collection payments --out $backup_path/payments.json
mongoexport --db brainpad_production --collection people --out $backup_path/people.json
mongoexport --db brainpad_production --collection reminders --out $backup_path/reminders.json
mongoexport --db brainpad_production --collection workouts --out $backup_path/workouts.json

# zip it up
zip $backup_path/$current_date $backup_path/*.json

# upload to s3
aws s3 cp $backup_path/$current_date.zip s3://$bucket_name --storage-class REDUCED_REDUNDANCY

# clean up
rm $backup_path/*.json
rm $backup_path/*.zip

echo 'finished. thanks for coming out'

schedule the backup

• crontab -e
• 1:00am every day: 0 1 * * * bin/db_backup >> log/db_backup.log
• list to make sure: crontab -l

prune old db backups

ruby aws sdk

• sudo gem install aws-sdk
• sudo gem install prettyprint

  script

• instructions aws

prune script

save this to bin/db_backup_prune.rb and make it executable chmod +x bin/db_backup_prune.rb

#!/usr/bin/env ruby

require 'active_support/all'
require 'date'
require 'aws-sdk'

bucket_name = 'raceweb-brainpad'
today = Date.today
month_start = today.beginning_of_month
last_month_end = month_start - 1.day
last_month_start = (today - 1.month).beginning_of_month
last_month_mask = last_month_start.strftime('%Y-%m')
s3 = Aws::S3::Resource.new()
bucket = s3.bucket(bucket_name)

p '##########################################################'
p "#{today}"
p "pruning backups from #{last_month_start} to #{last_month_end - 1.day}"
p "from S3 bucket #{bucket_name} using prefix: #{last_month_mask}"
p '##########################################################'

bucket.objects({ prefix: last_month_mask }).limit(50).each do |item|
  keeper = /#{last_month_end.strftime('%Y-%m-%d')}-01-00-\d{1,2}.zip/.match(item.key)
  p "name: #{item.key} #{!keeper ? 'DELETE' : 'KEEP'}"
  item.delete unless keeper
end
p ''

schedule it

• crontab -e
• 1:30am first day of every month: 30 1 1 * * bin/db_backup_prune.rb >> log/db_backup_prune.log
• list to make sure: crontab -l

restore from an s3 backup when necessary

restore script

save this to ~/bin/db_restore and make it executable chmod +x ~/bin/db_restore

#!/bin/bash

export PATH=/usr/local/bin:$PATH
bucket_name=raceweb-brainpad
backup_path=backup/current
file_name=$1
echo "##########################################################"
echo `date`
echo "restoring $file_name..."
echo "#########################################################"

# clean up stuff from previous runs
rm $backup_path/*.json
rm $backup_path/*.zip

# get the backup from s3
aws s3 cp s3://$bucket_name/$file_name $backup_path

# abort if the download didn't work
if [ -e $backup_path/$file_name ]; then
    echo "downloaded $file_name. unzipping it..."
    unzip $backup_path/$file_name
else
    echo "$backup_path/$file_name not found. does it exist on s3 in bucket: $bucket_name?"
    exit 1
fi

# import all the things
mongoimport --db brainpad_production --collection accounts --drop --file $backup_path/accounts.json
mongoimport --db brainpad_production --collection connections --drop --file $backup_path/connections.json
mongoimport --db brainpad_production --collection contacts --drop --file $backup_path/contacts.json
mongoimport --db brainpad_production --collection journals --drop --file $backup_path/journals.json
mongoimport --db brainpad_production --collection links --drop --file $backup_path/links.json
mongoimport --db brainpad_production --collection lookups --drop --file $backup_path/lookups.json
mongoimport --db brainpad_production --collection milestones --drop --file $backup_path/milestones.json
mongoimport --db brainpad_production --collection payments --drop --file $backup_path/payments.json
mongoimport --db brainpad_production --collection people --drop --file $backup_path/people.json
mongoimport --db brainpad_production --collection reminders --drop --file $backup_path/reminders.json
mongoimport --db brainpad_production --collection workouts --drop --file $backup_path/workouts.json

# clean  up
rm $backup_path/*.json
rm $backup_path/*.zip

echo 'finished. thanks for coming out'

run a restore

bin/db_restore 2017-02-21-01-00-01.zip >> log/db_restore.log