blog

Migrating PostgreSQL Databases From On-Prem to the Cloud Using AWS RDS

Ryan King

Published:

The following is a step-by-step guide to migrating your PostgreSQL on-premise databases to the cloud using Amazon RDS. For this tutorial, you will need to be running PostgreSQL 10 or later on a Centos/RHEL Linux server.

Step 1 — Create Associate Directories

#script directory
mkdir -p /opt/dba/postgres/migrate/
 
#backup directory
mkdir -p /dbbackup/
 
#log directory
mkdir -p /var/log/postgres/migrate/

Step 2 — Create A Backup Script

cd /opt/dba/postgres/migrate/
vi migrateDB_parallel.sh

Paste in the below contents:

chmod +x migrateDB_parallel.sh
 
#!/bin/bash
#set -e
set -u #exit immediately upon error
 
#################################################################################################################################
 
# Declare variables
 
migrate_date=$(date +'%m%d%Y')
log_dir=/var/log/postgres/migrate
 
psql_version=$1
 
source_host=$2
source_port=$3
source_database=$4
source_schema=$5
source_dbowner=$6
source_superuser=$7
source_jobs=$8
 
dest_host=$9
dest_port=${10}
dest_database=${11}
dest_dbowner=${12}
dest_superuser=${13}
dest_jobs=${14}
 
schema_only=${15}
no_owner=${16}
no_privileges=${17}
 
backup_dir=${18}/${psql_version}/${source_database}
 
test_run=${19}
 
if [[ ${source_schema} == n ]]; then
  sql_schema=""
  log="${log_dir}/${source_database}_${migrate_date}.log"
else
  sql_schema="--schema=${source_schema}"
  log="${log_dir}/${source_database}_${source_schema}_${migrate_date}.log"
fi
 
if [[ $schema_only == y ]]; then
  sql_schema_only="--schema-only"
else
  sql_schema_only=""
fi
 
if [[ $no_owner == y ]]; then
  sql_no_owner="--no-owner"
else
  sql_no_owner=""
fi
 
if [[ $no_privileges == y ]]; then
  sql_no_privileges="--no-privileges"
else
  sql_no_privileges=""
fi
 
BINDIR="/usr/pgsql-${psql_version}/bin"
 
#BINDIR="/usr/bin"
#log_dir=~/
 
mkdir -p ${log_dir}
 
 
################################################################################################################################
echo "${0} "  > $log
echo "" >> $log
echo "" >> $log
echo "$(date +'%m-%d-%Y %T') - DURATION: BEGIN"  >> $log
echo "" >> $log
echo "" >> $log
 
 
################################################################################################################################
# Display Parameters
echo " - psql_version:          $1"        >> $log
echo "                                    "        >> $log
echo " - source_host:                   $2"        >> $log
echo " - source_port:                   $3"        >> $log
echo " - source_database:               $4"        >> $log
echo " - source_schema:         $5"     >> $log
echo " - source_dbowner:                $6"        >> $log
echo " - source_superuser:              $7"        >> $log
echo " - source_jobs:                   $8"        >> $log
echo "                                    "        >> $log
echo " - dest_host:                     $9"        >> $log
echo " - dest_port:                     ${10}"        >> $log
echo " - dest_database:         ${11}"     >> $log
echo " - dest_dbowner:          ${12}"     >> $log
echo " - dest_superuser:                ${13}"     >> $log
echo " - dest_jobs:                     ${14}"     >> $log
echo "                                    "        >> $log
echo " - schema_only:                   ${15}"     >> $log
echo " - no_owner:                      ${16}"     >> $log
echo " - no_privileges:         ${17}"     >> $log
echo "                                    "        >> $log
echo " - sql_schema_only:               ${sql_schema_only}"     >> $log
echo " - sql_schema:                    ${sql_schema}"     >> $log
echo " - sql_no_owner:          ${sql_no_owner}"     >> $log
echo " - sql_no_privileges:             ${sql_no_privileges}"     >> $log
echo "                                    "        >> $log
echo " - test_run:                      ${19}"     >> $log
echo "                                    "        >> $log
echo " - BINDIR:                      ${BINDIR}" >> $log
echo " - backup_dir:                    ${backup_dir}"     >> $log
echo " - log_file:                    ${log}"    >> $log
echo "" >> $log
echo "" >> $log
echo "" >> $log
 
# vi $log; exit 99;
 
#################################################################################################################################
# Backup database
echo "######################################################################################################"  >> $log
echo "$(date +'%m-%d-%Y %T') - beginning database backup..."  >> $log
echo "" >> $log
echo "" >> $log
 
if [[ ${source_schema} == n ]]; then
 
        echo "# Gather ${source_host}:${source_database} db size..." >> $log
        ${BINDIR}/psql -p ${source_port} postgres -h${source_host} -U${source_superuser} -c"l+ ${source_database}" >> $log 2>&1
 
else
 
        echo "# Gather ${source_host}:${source_database}.${source_schema} schema size..." >> $log
        ${BINDIR}/psql -p ${source_port} -h${source_host} ${source_database} -U${source_superuser} -c"
                SELECT schema_name, pg_size_pretty(sum(table_size)::bigint) as diskSpace, (sum(table_size) / pg_database_size(current_database())) * 100 as percent
                FROM (
                        SELECT pgn.nspname as schema_name,pg_relation_size(pgc.oid) as table_size
                        FROM   pg_catalog.pg_class pgc
                        JOIN pg_catalog.pg_namespace pgn ON relnamespace = pgn.oid
                        WHERE pgn.nspname='${source_schema}'
                     ) t
                GROUP BY schema_name" >> $log 2>&1
 
fi
 
echo "" >> $log
echo "" >> $log
 
 
echo "# Refresh directories..." >> $log
echo "mkdir -p ${backup_dir}" >> $log 2>&1
echo "rm    -f ${backup_dir}/*" >> $log 2>&1
if [[ $test_run == n ]]; then
      mkdir -p ${backup_dir}  >> $log 2>&1
      rm    -f ${backup_dir}/*  >> $log 2>&1
fi
echo "" >> $log
echo "" >> $log
 
 
# vi $log; exit 99;
 
echo "# Backup database..." >> $log
echo "${BINDIR}/pg_dump -v ${source_database} -h ${source_host} -p ${source_port} -U ${source_dbowner} -Fd -j ${source_jobs} -f ${backup_dir} ${sql_schema} ${sql_schema_only} ${sql_no_owner} ${sql_no_privileges}" >> $log 2>&1
if [[ $test_run == n ]]; then
      ${BINDIR}/pg_dump -v ${source_database} -h ${source_host} -p ${source_port} -U ${source_dbowner} -Fd -j ${source_jobs} -f ${backup_dir} ${sql_schema} ${sql_schema_only} ${sql_no_owner} ${sql_no_privileges}  >> $log 2>&1
fi
echo "" >> $log
echo "" >> $log
echo "$(date +'%m-%d-%Y %T') - completed database backup..." >> $log
echo "" >> $log
echo "" >> $log
echo "" >> $log
 
 
#################################################################################################################################
echo "######################################################################################################"  >> $log
echo "$(date +'%m-%d-%Y %T') - beginning database restore..."  >> $log
echo "" >> $log
echo "" >> $log
 
 
 
 
if [[ ${source_schema} == n ]]; then
 
        echo "# Grant ${dest_dbowner} createdb..." >> $log
        echo "${BINDIR}/psql -E -e postgres -h${dest_host} -p${dest_port} -U${dest_superuser} -c"alter role ${dest_dbowner} createdb"" >> $log 2>&1
        if [[ $test_run == n ]]; then
              ${BINDIR}/psql -E -e postgres -h${dest_host} -p${dest_port} -U${dest_superuser} -c "alter role ${dest_dbowner} createdb"   >> $log 2>&1
        fi
        echo "" >> $log
        echo "" >> $log
 
        echo "# Create database..." >> $log
        echo "${BINDIR}/psql -E -e postgres -h${dest_host} -p${dest_port} -U${dest_dbowner} -c"create database ${dest_database}"" >> $log 2>&1
        if [[ $test_run == n ]]; then
              ${BINDIR}/psql -E -e postgres -h${dest_host} -p${dest_port} -U${dest_dbowner} -c "create database ${dest_database}"   >> $log 2>&1
        fi
        echo "" >> $log
        echo "" >> $log
        #vi $log; exit 99;
 
else
 
        echo "# Create schema..." >> $log
        echo "${BINDIR}/psql -E -e -h${dest_host} -p${dest_port} ${dest_database} -U${dest_dbowner} -c"create schema ${source_schema}"" >> $log 2>&1
        if [[ $test_run == n ]]; then
              ${BINDIR}/psql -E -e -h${dest_host} -p${dest_port} ${dest_database} -U${dest_dbowner} -c "create schema ${source_schema}"   >> $log 2>&1
        fi
        echo "" >> $log
        echo "" >> $log
        #vi $log; exit 99;
 
fi
 
echo "# Restore database..." >> $log
echo "${BINDIR}/pg_restore -v -d ${dest_database} -h${dest_host} -p${dest_port} -U ${dest_dbowner} -j ${dest_jobs} -Fd ${backup_dir} ${sql_schema} ${sql_schema_only} ${sql_no_owner} ${sql_no_privileges}" >> $log 2>&1
if [[ $test_run == n ]]; then
      ${BINDIR}/pg_restore -v -d ${dest_database} -h${dest_host} -p${dest_port} -U ${dest_dbowner} -j ${dest_jobs} -Fd ${backup_dir} ${sql_schema} ${sql_schema_only} ${sql_no_owner} ${sql_no_privileges}  >> $log 2>&1
fi
echo "" >> $log
echo "" >> $log
echo "$(date +'%m-%d-%Y %T') - completed database restore..." >> $log
echo "" >> $log
echo "" >> $log
 
if [[ $source_schema != n ]]; then
        ${BINDIR}/psql -h${dest_host} -p${dest_port} ${dest_database} -U${dest_dbowner} -c "dn+ ${source_schema}"   >> $log 2>&1
        ${BINDIR}/psql -h${dest_host} -p${dest_port} ${dest_database} -U${dest_dbowner} -c "dt+ ${source_schema}.*" >> $log 2>&1
fi
echo "" >> $log
echo "" >> $log
echo "" >> $log
echo "" >> $log
 
echo "$(date +'%m-%d-%Y %T') - DURATION: END" >> $log
vi $log

Step 3 — Execute a Test Run for all Schemas and Review Commands

To properly prepare for the migration, gather all details on exactly what to backup and what to restore. This data will serve as parameters to be passed into the shell script above. In this example, we will migrate a full database including all its schemas passing in the following parameters:

Parameters Command (Param Values)
psql_version=$1
source_host=$2
source_port=$3
source_database=$4
source_schema=$5
source_dbowner=$6
source_superuser=$7
source_jobs=$8
dest_host=$9
dest_port=${10}
dest_database=${11}
dest_dbowner=${12}
dest_superuser=${13}
dest_jobs=${14}
schema_only=${15}
no_owner=${16}
no_privileges=${17}
backup_dir=${18}/${psql_version}/${source_database}
test_run=${19}		 /opt/dba/migrate/migrateDB_parallel.sh
10
onpremdbserver.domain.com
5432
source_database
n
source_appuser
postgres
100
dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
5432
dest_database_newdb
dest_rds_superuser
dest_rds_superuser
120
n
y
y
“/dbbackup”
Y

Make Sure to Review the Commands Before Execution

05-24-2019 22:14:08 - DURATION: BEGIN
  
 - psql_version:        10
                                    
 - source_host:        onpremdbserver.domain.com
 - source_port:        5432
 - source_database:        source_database
 - source_schema:        n
 - source_dbowner:        source_appuser
 - source_superuser:        postgres
 - source_jobs:        100
                                    
 - dest_host:            dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
 - dest_port:            5432
 - dest_database:        dest_database_newdb
 - dest_dbowner:        dest_rds_superuser
 - dest_superuser:        dest_rds_superuser
 - dest_jobs:            120
                                    
 - schema_only:        n
 - no_owner:            y
 - no_privileges:        y
                                    
 - sql_schema_only:               
 - sql_schema:                    
 - sql_no_owner:        --no-owner
 - sql_no_privileges:        --no-privileges
                                    
 - test_run:            y
                                    
 - BINDIR:            /usr/pgsql-10/bin
 - backup_dir:            /dbbackup/10/source_database
 - log_file:            /var/log/postgres/migrate/source_database_05242019.log
  
 
######################################################################################################
05-24-2019 22:14:08 - beginning database backup...
  
# Gather onpremdbserver.domain.com:source_database db size...
                                                        List of databases
      Name       |     Owner      | Encoding |   Collate   |    Ctype    | Access privileges |  Size   | Tablespace | Description
-----------------+----------------+----------+-------------+-------------+-------------------+---------+------------+-------------
 source_database | source_appuser | UTF8     | en_US.UTF-8 | en_US.UTF-8 |                   | 7992 kB | pg_default |
(1 row)
 
  
# Refresh directories...
mkdir -p /dbbackup/10/source_database
rm    -f /dbbackup/10/source_database/*
 
 
# Backup database...
/usr/pgsql-10/bin/pg_dump -v source_database -h onpremdbserver.domain.com -p 5432 -U source_appuser -Fd -j 100 -f /dbbackup/9.6/source_database   --no-owner --no-privileges
  
05-24-2019 22:14:09 - completed database backup...
 
  
######################################################################################################
05-24-2019 22:14:09 - beginning database restore...
  
# Grant dest_rds_superuser createdb...
/usr/pgsql-9.6/bin/psql -E -e postgres -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -Udest_rds_superuser -c"alter role dest_rds_superuser createdb"
  
# Create database...
/usr/pgsql-9.6/bin/psql -E -e postgres -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -Udest_rds_superuser -c"create database dest_database_newdb"
  
# Restore database...
/usr/pgsql-9.6/bin/pg_restore -v -d dest_database_newdb -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -U dest_rds_superuser -j 120 -Fd /dbbackup/9.6/source_database   --no-owner --no-privileges
  
05-24-2019 22:14:09 - completed database restore...

Step 4 — Execute Script (All Schemas)

Note: Pass in “n” for all schemas:

Parameters Command (Param Values)
psql_version=$1
source_host=$2
source_port=$3
source_database=$4
source_schema=$5
source_dbowner=$6
source_superuser=$7
source_jobs=$8
dest_host=$9
dest_port=${10}
dest_database=${11}
dest_dbowner=${12}
dest_superuser=${13}
dest_jobs=${14}
schema_only=${15}
no_owner=${16}
no_privileges=${17}
backup_dir=${18}/${psql_version}/${source_database}
test_run=${19}		 /opt/dba/migrate/migrateDB_parallel.sh
10
onpremdbserver.domain.com
5432
source_database
n
source_appuser
postgres
100
dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
5432
dest_database_newdb
dest_rds_superuser
dest_rds_superuser
120
n
y
y
“/dbbackup”
Y

Output

05-24-2019 23:16:40 - DURATION: BEGIN
 
 - psql_version:              10    
 - source_host:               onpremdbserver.domain.com
 - source_port:               5432
 - source_database:           source_database
 - source_schema:        n
 - source_dbowner:            source_appuser
 - source_superuser:          postgres
 - source_jobs:               100
                                    
 - dest_host:            dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
 - dest_port:            5432
 - dest_database:             dest_database_newdb
 - dest_dbowner:              dest_rds_superuser
 - dest_superuser:            dest_rds_superuser
 - dest_jobs:                 120
                                    
 - schema_only:               n
 - no_owner:                  y
 - no_privileges:             y
                                    
 - sql_schema_only:               
 - sql_schema:                    
 - sql_no_owner:              --no-owner
 - sql_no_privileges:         --no-privileges
                                    
 - test_run:                  n
                                    
 - BINDIR:            /usr/pgsql-10/bin
 - backup_dir:                /dbbackup/10/source_database
 - log_file:                  /var/log/postgres/migrate/source_database_05242019.log
 
  
######################################################################################################
05-24-2019 23:16:40 - beginning database backup...
 
 # Gather onpremdbserver.domain.com db size...
                                                        List of databases
      Name       |     Owner      | Encoding |   Collate   |    Ctype    | Access privileges |  Size   | Tablespace | Description
-----------------+----------------+----------+-------------+-------------+-------------------+---------+------------+-------------
 source_database | source_appuser | UTF8     | en_US.UTF-8 | en_US.UTF-8 |                   | 7992 kB | pg_default |
(1 row)
  
# Refresh directories...
mkdir -p /dbbackup/10/source_database
rm    -f /dbbackup/10/source_database/*
 
 # Backup database...
/usr/pgsql-10/bin/pg_dump -v source_database -h onpremdbserver.domain.com -p 5432 -U source_appuser -Fd -j 100 -f /dbbackup/10/source_database   --no-owner --no-privileges
pg_dump: last built-in OID is 16383
pg_dump: reading extensions
pg_dump: identifying extension members
pg_dump: reading schemas
pg_dump: reading user-defined tables
pg_dump: reading user-defined functions
pg_dump: reading user-defined types
pg_dump: reading procedural languages
pg_dump: reading user-defined aggregate functions
pg_dump: reading user-defined operators
pg_dump: reading user-defined access methods
pg_dump: reading user-defined operator classes
pg_dump: reading user-defined operator families
pg_dump: reading user-defined text search parsers
pg_dump: reading user-defined text search templates
pg_dump: reading user-defined text search dictionaries
pg_dump: reading user-defined text search configurations
pg_dump: reading user-defined foreign-data wrappers
pg_dump: reading user-defined foreign servers
pg_dump: reading default privileges
pg_dump: reading user-defined collations
pg_dump: reading user-defined conversions
pg_dump: reading type casts
pg_dump: reading transforms
pg_dump: reading table inheritance information
pg_dump: reading event triggers
pg_dump: finding extension tables
pg_dump: finding inheritance relationships
pg_dump: reading column info for interesting tables
pg_dump: finding the columns and types of table "source_schema.table1"
pg_dump: finding the columns and types of table "source_schema.table2"
pg_dump: finding the columns and types of table "source_schema.table3"
pg_dump: finding default expressions of table "source_schema.table3"
pg_dump: flagging inherited columns in subtables
pg_dump: reading indexes
pg_dump: reading extended statistics
pg_dump: reading constraints
pg_dump: reading triggers
pg_dump: reading rewrite rules
pg_dump: reading policies
pg_dump: reading row security enabled for table "source_schema.table1"
pg_dump: reading policies for table "source_schema.table1"
pg_dump: reading row security enabled for table "source_schema.table2"
pg_dump: reading policies for table "source_schema.table2"
pg_dump: reading row security enabled for table "source_schema.table3_id_seq"
pg_dump: reading policies for table "source_schema.table3_id_seq"
pg_dump: reading row security enabled for table "source_schema.table3"
pg_dump: reading policies for table "source_schema.table3"
pg_dump: reading publications
pg_dump: reading publication membership
pg_dump: reading publication membership for table "source_schema.table1"
pg_dump: reading publication membership for table "source_schema.table2"
pg_dump: reading publication membership for table "source_schema.table3"
pg_dump: reading subscriptions
pg_dump: reading large objects
pg_dump: reading dependency data
pg_dump: saving encoding = UTF8
pg_dump: saving standard_conforming_strings = on
pg_dump: saving search_path =
pg_dump: saving database definition
pg_dump: dumping contents of table "source_schema.table3"
pg_dump: finished item 3800 TABLE DATA table3
pg_dump: dumping contents of table "source_schema.table1"
pg_dump: finished item 3797 TABLE DATA table1
pg_dump: dumping contents of table "source_schema.table2"
pg_dump: finished item 3798 TABLE DATA table2
  
05-24-2019 23:16:41 - completed database backup...
   ######################################################################################################
05-24-2019 23:16:41 - beginning database restore...
  
# Grant dest_rds_superuser createdb...
/usr/pgsql-10/bin/psql -E -e postgres -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -Udest_rds_superuser -c"alter role dest_rds_superuser createdb"
alter role dest_rds_superuser createdb
ALTER ROLE
  
# Create database...
/usr/pgsql-10/bin/psql -E -e postgres -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -Udest_rds_superuser -c"create database dest_database_newdb"
create database dest_database_newdb
CREATE DATABASE
  
# Restore database...
/usr/pgsql-10/bin/pg_restore -v -d dest_database_newdb -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -U dest_rds_superuser -j 120 -Fd /dbbackup/10/source_database   --no-owner --no-privileges
pg_restore: connecting to database for restore
pg_restore: processing item 3803 ENCODING ENCODING
pg_restore: processing item 3804 STDSTRINGS STDSTRINGS
pg_restore: processing item 3805 SEARCHPATH SEARCHPATH
pg_restore: processing item 3806 DATABASE source_database
pg_restore: processing item 3 SCHEMA public
pg_restore: creating SCHEMA "public"
pg_restore: processing item 3807 COMMENT SCHEMA public
pg_restore: creating COMMENT "SCHEMA public"
pg_restore: processing item 6 SCHEMA source_schema
pg_restore: creating SCHEMA "source_schema"
pg_restore: processing item 1 EXTENSION plpgsql
pg_restore: creating EXTENSION "plpgsql"
pg_restore: processing item 3808 COMMENT EXTENSION plpgsql
pg_restore: creating COMMENT "EXTENSION plpgsql"
pg_restore: [archiver (db)] Error while PROCESSING TOC:
pg_restore: [archiver (db)] Error from TOC entry 3808; 0 0 COMMENT EXTENSION plpgsql
pg_restore: [archiver (db)] could not execute query: ERROR:  must be owner of extension plpgsql
    Command was: COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
 
 
 
pg_restore: processing item 197 TABLE table1
pg_restore: creating TABLE "source_schema.table1"
pg_restore: processing item 198 TABLE table2
pg_restore: creating TABLE "source_schema.table2"
pg_restore: processing item 200 TABLE table3
pg_restore: creating TABLE "source_schema.table3"
pg_restore: processing item 199 SEQUENCE table3_id_seq
pg_restore: creating SEQUENCE "source_schema.table3_id_seq"
pg_restore: processing item 3809 SEQUENCE OWNED BY table3_id_seq
pg_restore: creating SEQUENCE OWNED BY "source_schema.table3_id_seq"
pg_restore: processing item 3675 DEFAULT table3 id
pg_restore: creating DEFAULT "source_schema.table3 id"
pg_restore: entering main parallel loop
pg_restore: launching item 3797 TABLE DATA table1
pg_restore: launching item 3798 TABLE DATA table2
pg_restore: launching item 3800 TABLE DATA table3
pg_restore: launching item 3810 SEQUENCE SET table3_id_seq
pg_restore: processing data for table "source_schema.table2"
pg_restore: executing SEQUENCE SET table3_id_seq
pg_restore: finished item 3810 SEQUENCE SET table3_id_seq
pg_restore: processing data for table "source_schema.table1"
pg_restore: processing data for table "source_schema.table3"
pg_restore: finished item 3797 TABLE DATA table1
pg_restore: finished item 3798 TABLE DATA table2
pg_restore: finished item 3800 TABLE DATA table3
pg_restore: finished main parallel loop
WARNING: errors ignored on restore: 1
  
05-24-2019 23:16:43 - completed database restore...
  
05-24-2019 23:16:43 - DURATION: END

Step 5 — Execute a Test Run for the Schema

Parameters Command (Param Values)
psql_version=$1
source_host=$2
source_port=$3
source_database=$4
source_schema=$5
source_dbowner=$6
source_superuser=$7
source_jobs=$8
dest_host=$9
dest_port=${10}
dest_database=${11}
dest_dbowner=${12}
dest_superuser=${13}
dest_jobs=${14}
schema_only=${15}
no_owner=${16}
no_privileges=${17}
backup_dir=${18}/${psql_version}/${source_database}
test_run=${19}		 /opt/dba/migrate/migrateDB_parallel.sh
10
onpremdbserver.domain.com
5432
source_database
source_schema
source_appuser
postgres
100
dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
5432
dest_database_newdb
dest_rds_superuser
dest_rds_superuser
120
n
y
y
“/dbbackup”
y

Again, Make Sure to Review the Commands Before Execution

05-24-2019 22:50:48 - DURATION: BEGIN
  
 - psql_version:        10
                                    
 - source_host:               onpremdbserver.domain.com
 - source_port:               5432
 - source_database:           source_database
 - source_schema:        source_schema
 - source_dbowner:        source_appuser
 - source_superuser:          postgres
 - source_jobs:               100
                                    
 - dest_host:                 dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
 - dest_port:                 5432
 - dest_database:             dest_database_newdb
 - dest_dbowner:              dest_rds_superuser
 - dest_superuser:            dest_rds_superuser
 - dest_jobs:                 120
                                    
 - schema_only:               n
 - no_owner:                  y
 - no_privileges:             y
                                    
 - sql_schema_only:               
 - sql_schema:                --schema=source_schema
 - sql_no_owner:              --no-owner
 - sql_no_privileges:         --no-privileges
                                    
 - test_run:                  y
                                    
 - BINDIR:                    /usr/pgsql-10/bin
 - backup_dir:                /dbbackup/10/source_database
 - log_file:            /var/log/postgres/migrate/source_database_source_schema_05242019.log
 
  
######################################################################################################
05-24-2019 22:50:48 - beginning database backup...
  
# Gather onpremdbserver.domain.com:source_database.source_schema schema size...
  schema_name  | diskspace  |        percent         
---------------+------------+------------------------
 source_schema | 8192 bytes | 0.10010060159337380900
(1 row)
 
 
# Refresh directories...
mkdir -p /dbbackup/10/source_database
rm    -f /dbbackup/10/source_database/*
  
# Backup database...
/usr/pgsql-10/bin/pg_dump -v source_database -h onpremdbserver.domain.com -p 5432 -U source_appuser -Fd -j 100 -f /dbbackup/10/source_database --schema=source_schema  --no-owner --no-privileges
  
05-24-2019 22:50:49 - completed database backup...
  
######################################################################################################
05-24-2019 22:50:49 - beginning database restore...
  
# Create schema...
/usr/pgsql-10/bin/psql -E -e -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 dest_database_newdb -Udest_rds_superuser -c"create schema source_schema"
  
# Restore database...
/usr/pgsql-10/bin/pg_restore -v -d dest_database_newdb -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -U dest_rds_superuser -j 120 -Fd /dbbackup/10/source_database --schema=source_schema  --no-owner --no-privileges
  
05-24-2019 22:50:49 - completed database restore...
  
                           List of schemas
     Name      |       Owner        | Access privileges | Description
---------------+--------------------+-------------------+-------------
 source_schema | dest_rds_superuser |                   |
(1 row)
 
                               List of relations
    Schema     |  Name  | Type  |       Owner        |    Size    | Description
---------------+--------+-------+--------------------+------------+-------------
 source_schema | table1 | table | dest_rds_superuser | 0 bytes    |
 source_schema | table2 | table | dest_rds_superuser | 0 bytes    |
 source_schema | table3 | table | dest_rds_superuser | 8192 bytes |
(3 rows)
 
 
 
05-24-2019 22:50:49 - DURATION: END

Step 6 — Execute Script (One Schema)

Parameters Command (Param Values)
psql_version=$1
source_host=$2
source_port=$3
source_database=$4
source_schema=$5
source_dbowner=$6
source_superuser=$7
source_jobs=$8
dest_host=$9
dest_port=${10}
dest_database=${11}
dest_dbowner=${12}
dest_superuser=${13}
dest_jobs=${14}
schema_only=${15}
no_owner=${16}
no_privileges=${17}
backup_dir=${18}/${psql_version}/${source_database}
test_run=${19}		 /opt/dba/migrate/migrateDB_parallel.sh
10
onpremdbserver.domain.com
5432
source_database
source_schema
source_appuser
postgres
100
dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
5432
dest_database_newdb
dest_rds_superuser
dest_rds_superuser
120
n
y
y
“/dbbackup”
n

Note: Pass in “source_schema” for schema name

Review the Commands Before Execution

05-24-2019 23:22:13 - DURATION: BEGIN
  
 - psql_version:              10
                                    
 - source_host:               onpremdbserver.domain.com
 - source_port:            5432
 - source_database:           source_database
 - source_schema:             source_schema
 - source_dbowner:            source_appuser
 - source_superuser:          postgres
 - source_jobs:               100
                                    
 - dest_host:                 dest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com
 - dest_port:                 5432
 - dest_database:             dest_database_newdb
 - dest_dbowner:              dest_rds_superuser
 - dest_superuser:            dest_rds_superuser
 - dest_jobs:                 120
                                    
 - schema_only:               n
 - no_owner:                  y
 - no_privileges:             y
                                    
 - sql_schema_only:               
 - sql_schema:                --schema=source_schema
 - sql_no_owner:        --no-owner
 - sql_no_privileges:        --no-privileges
                                    
 - test_run:                  n
                                    
 - BINDIR:                    /usr/pgsql-10/bin
 - backup_dir:                /dbbackup/10/source_database
 - log_file:            /var/log/postgres/migrate/source_database_source_schema_05242019.log
 
 
######################################################################################################
05-24-2019 23:22:13 - beginning database backup...
 
 
# Gather onpremdbserver.domain.com:source_database.source_schema schema size...
  schema_name  | diskspace  |        percent         
---------------+------------+------------------------
 source_schema | 8192 bytes | 0.10010060159337380900
(1 row)
 
 
# Refresh directories...
mkdir -p /dbbackup/10/source_database
rm    -f /dbbackup/10/source_database/*
  
# Backup database...
/usr/pgsql-10/bin/pg_dump -v source_database -h onpremdbserver.domain.com -p 5432 -U source_appuser -Fd -j 100 -f /dbbackup/10/source_database --schema=source_schema  --no-owner --no-privileges
pg_dump: last built-in OID is 16383
pg_dump: reading extensions
pg_dump: identifying extension members
pg_dump: reading schemas
pg_dump: reading user-defined tables
pg_dump: reading user-defined functions
pg_dump: reading user-defined types
pg_dump: reading procedural languages
pg_dump: reading user-defined aggregate functions
pg_dump: reading user-defined operators
pg_dump: reading user-defined access methods
pg_dump: reading user-defined operator classes
pg_dump: reading user-defined operator families
pg_dump: reading user-defined text search parsers
pg_dump: reading user-defined text search templates
pg_dump: reading user-defined text search dictionaries
pg_dump: reading user-defined text search configurations
pg_dump: reading user-defined foreign-data wrappers
pg_dump: reading user-defined foreign servers
pg_dump: reading default privileges
pg_dump: reading user-defined collations
pg_dump: reading user-defined conversions
pg_dump: reading type casts
pg_dump: reading transforms
pg_dump: reading table inheritance information
pg_dump: reading event triggers
pg_dump: finding extension tables
pg_dump: finding inheritance relationships
pg_dump: reading column info for interesting tables
pg_dump: finding the columns and types of table "source_schema.table1"
pg_dump: finding the columns and types of table "source_schema.table2"
pg_dump: finding the columns and types of table "source_schema.table3"
pg_dump: finding default expressions of table "source_schema.table3"
pg_dump: flagging inherited columns in subtables
pg_dump: reading indexes
pg_dump: reading extended statistics
pg_dump: reading constraints
pg_dump: reading triggers
pg_dump: reading rewrite rules
pg_dump: reading policies
pg_dump: reading row security enabled for table "source_schema.table1"
pg_dump: reading policies for table "source_schema.table1"
pg_dump: reading row security enabled for table "source_schema.table2"
pg_dump: reading policies for table "source_schema.table2"
pg_dump: reading row security enabled for table "source_schema.table3_id_seq"
pg_dump: reading policies for table "source_schema.table3_id_seq"
pg_dump: reading row security enabled for table "source_schema.table3"
pg_dump: reading policies for table "source_schema.table3"
pg_dump: reading publications
pg_dump: reading publication membership
pg_dump: reading publication membership for table "source_schema.table1"
pg_dump: reading publication membership for table "source_schema.table2"
pg_dump: reading publication membership for table "source_schema.table3"
pg_dump: reading subscriptions
pg_dump: reading dependency data
pg_dump: saving encoding = UTF8
pg_dump: saving standard_conforming_strings = on
pg_dump: saving search_path =
pg_dump: dumping contents of table "source_schema.table1"
pg_dump: dumping contents of table "source_schema.table3"
pg_dump: finished item 3797 TABLE DATA table1
pg_dump: finished item 3800 TABLE DATA table3
pg_dump: dumping contents of table "source_schema.table2"
pg_dump: finished item 3798 TABLE DATA table2
 
 
05-24-2019 23:22:14 - completed database backup...
 
 
######################################################################################################
05-24-2019 23:22:14 - beginning database restore...
 
 # Create schema...
/usr/pgsql-10/bin/psql -E -e -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 dest_database_newdb -Udest_rds_superuser -c"create schema source_schema"
create schema source_schema
CREATE SCHEMA
  
# Restore database...
/usr/pgsql-10/bin/pg_restore -v -d dest_database_newdb -hdest_dbinstance.cluster-awsrdsguid.us-east-1.rds.amazonaws.com -p5432 -U dest_rds_superuser -j 120 -Fd /dbbackup/10/source_database --schema=source_schema  --no-owner --no-privileges
pg_restore: connecting to database for restore
pg_restore: processing item 3803 ENCODING ENCODING
pg_restore: processing item 3804 STDSTRINGS STDSTRINGS
pg_restore: processing item 3805 SEARCHPATH SEARCHPATH
pg_restore: processing item 6 SCHEMA source_schema
pg_restore: processing item 197 TABLE table1
pg_restore: creating TABLE "source_schema.table1"
pg_restore: processing item 198 TABLE table2
pg_restore: creating TABLE "source_schema.table2"
pg_restore: processing item 200 TABLE table3
pg_restore: creating TABLE "source_schema.table3"
pg_restore: processing item 199 SEQUENCE table3_id_seq
pg_restore: creating SEQUENCE "source_schema.table3_id_seq"
pg_restore: processing item 3806 SEQUENCE OWNED BY table3_id_seq
pg_restore: creating SEQUENCE OWNED BY "source_schema.table3_id_seq"
pg_restore: processing item 3675 DEFAULT table3 id
pg_restore: creating DEFAULT "source_schema.table3 id"
pg_restore: entering main parallel loop
pg_restore: launching item 3797 TABLE DATA table1
pg_restore: launching item 3798 TABLE DATA table2
pg_restore: launching item 3800 TABLE DATA table3
pg_restore: launching item 3807 SEQUENCE SET table3_id_seq
pg_restore: pg_restore: processing data for table "source_schema.table2"
processing data for table "source_schema.table3"
pg_restore: executing SEQUENCE SET table3_id_seq
pg_restore: processing data for table "source_schema.table1"
pg_restore: finished item 3807 SEQUENCE SET table3_id_seq
pg_restore: finished item 3798 TABLE DATA table2
pg_restore: finished item 3800 TABLE DATA table3
pg_restore: finished item 3797 TABLE DATA table1
pg_restore: finished main parallel loop
 
 
05-24-2019 23:22:14 - completed database restore...
  
                           List of schemas
     Name      |       Owner        | Access privileges | Description
---------------+--------------------+-------------------+-------------
 source_schema | dest_rds_superuser |                   |
(1 row)
                                List of relations
    Schema     |  Name  | Type  |       Owner        |    Size    | Description
---------------+--------+-------+--------------------+------------+-------------
 source_schema | table1 | table | dest_rds_superuser | 0 bytes    |
 source_schema | table2 | table | dest_rds_superuser | 0 bytes    |
 source_schema | table3 | table | dest_rds_superuser | 8192 bytes |
(3 rows)
 
 
05-24-2019 23:22:14 - DURATION: END

Step 7 — Clean Up

After validating from the end user, remove any older databases and/or schemas. You have now migrated your database from on prem to AWS RDS! In the next part, I will discuss “Common Errors when Migrating PostgreSQL Databases From On-Prem to AWS RDS”.

Recommended

A MySQL backup and recovery tutorial on K8s using the MySQL Operator for Kubernetes

October 17, 2024 Divine Odazie

Implementing the golden backup rule for your databases

September 30, 2024 Krzysztof Ksiazek

An overview of running your databases on and with Kubernetes

September 3, 2024 Divine Odazie

The Sovereign Cloud brief

August 22, 2024 Marisa Crous

Subscribe below to be notified of fresh posts