updates from manomano

.gitignore

@@ -1 +1,2 @@
 *config
+metadata/*

.shellcheckrc

@@ -0,0 +1 @@
+disable=SC2086,SC2046,SC2034,SC2162,SC2196,SC2317,SC2164,SC2316,SC1090,SC2035

SCRIPT_TEMPLATE.sh

@@ -0,0 +1,288 @@
+#!/bin/bash
+
+# Exit codes:
+#  1 : 
+#  2 : 
+#  3 : 
+#  4 : 
+
+
+########################################################################
+# INIT
+########################################################################
+
+CONFIGFILE="$(dirname "$(readlink -f "$0")")/$(basename "$(readlink -f "$0")" .sh).config"
+
+########################################################################
+#/INIT
+########################################################################
+
+
+
+########################################################################
+#
+# CONSTANTS
+#
+########################################################################
+
+# colors
+BOLD="\e[1m"
+GREEN="\e[32m"
+LIGHTGREEN="${BOLD}${GREEN}"
+#RED="\033[1;31m"
+LIGHTRED="\033[1;31m"
+BLUE="\e[34m"
+LIGHTBLUE="${BOLD}${BLUE}"
+YELLOW="\e[33m"
+LIGHTYELLOW="${BOLD}${YELLOW}"
+#WHITE="\033[0;37m"
+RESET="\033[0;00m"
+
+NOW="$(date +%Y%m%d%H%M%S)"
+
+# set to 0 to output messages without colors
+NOCOLOR=0
+
+# set to 0 for debugging
+DEBUG=0
+#DEBUG=1
+
+EXITCODE=0
+
+########################################################################
+#
+# / CONSTANTS
+#
+########################################################################
+
+
+
+
+########################################################################
+#
+# VARIABLES
+#
+########################################################################
+
+
+SCRIPTLOG="$(dirname "$(readlink -f "$0")")/logs/$(basename "$0" .sh)_script_${NOW}.log"
+SCRIPTLOGERR="$(dirname "$(readlink -f "$0")")/logs/$(basename "$0" .sh)_script_${NOW}.err"
+
+# delete logs older than PURGE_LOG_TIME days
+PURGE_LOG_TIME=30
+
+########################################################################
+#
+# / VARIABLES
+#
+########################################################################
+
+
+
+########################################################################
+#
+# FUNCTIONS
+#
+########################################################################
+
+
+usage()
+{
+    printf "%s${LIGHTRED}USAGE:${RESET}
+    $0 -u USERNAME -t TEMPLAGE_FILE [-h] [-D]
+    
+    -u USERNAME         ${LIGHTGREEN}USERNAME${RESET} to be created
+    -t TEMPLATE_FILE    ${LIGHTGREEN}TEMPLATE_FILE${RESET} to be used (must exist)
+    -p PASSWORD         ${LIGHTGREEN}PASSWORD${RESET} for the user
+    -h                  this help
+    -D                  DEBUG mode
+"
+    # VERY INITIAL CHECKS
+}
+
+
+no_colors_output()
+{
+    if [[ ${NOCOLOR} -eq 0 ]] ; then
+        # colors
+        BOLD=""
+        GREEN=""
+        LIGHTGREEN=""
+        LIGHTRED=""
+        BLUE=""
+        LIGHTBLUE=""
+        YELLOW=""
+        LIGHTYELLOW=""
+        RESET=""
+    fi
+}
+
+
+printmsg()
+{
+   echo -e "$*"
+}
+
+output_log()
+{
+    if [[ "${QUIETOUTPUT}" == true ]]; then
+        printmsg "$*" >> "${OUTPUTFILE}"
+    else
+        printmsg "$*" | tee -a "${OUTPUTFILE}"
+    fi
+}
+
+abort_message()
+{
+    local ABORTCODE=1
+    [[ ${EXITCODE} -ne 0 ]] && ABORTCODE=${EXITCODE}
+    printmsg "${LIGHTRED}ERROR${RESET}: $*"
+    exit ${ABORTCODE}
+}
+
+# debug_me uses variable ${DEBUG}
+debug_me()
+{
+    if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
+        printmsg "${LIGHTBLUE}[DEBUG]${RESET}: $*"
+    fi
+}
+
+warning_message()
+{
+    printmsg "${LIGHTYELLOW}[WARNING]${RESET}: $*"
+}
+
+ok_message()
+{
+    printmsg "${LIGHTGREEN}[OK]${RESET}: $*"
+}
+
+nice_countdown()
+{
+    local WAIT_TIME=$1
+    local DEFAULT_WAIT_TIME=10
+    if [[ ! ${WAIT_TIME} ]] ; then
+        WAIT_TIME=${DEFAULT_WAIT_TIME}
+    fi
+    local WARNING=$((WAIT_TIME*2/3))
+    local CRITICAL=$((WAIT_TIME/3))
+    for ((x=WAIT_TIME; x>0; x--)) ; do
+        if [[ ${x} -gt ${WARNING} ]] ; then
+            MSG_COLOR=${LIGHTGREEN}
+        elif [[ ${x} -gt ${CRITICAL} ]] ; then
+            MSG_COLOR=${LIGHTYELLOW}
+        else
+            MSG_COLOR=${LIGHTRED}
+        fi
+        printf "\r${MSG_COLOR}%s${RESET} .." "${x}"
+        sleep 1
+    done
+    echo
+}
+
+# ssh_it uses variable ${DEBUG}
+ssh_it()
+{
+    local RESSULT=99
+    if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
+        ${SSHIT} "$*"
+        RESSULT=$?
+    else
+        ${SSHIT} "$*" 2>/dev/null
+        RESSULT=$?
+    fi
+    return "${RESSULT}"
+}
+
+log_maintenance()
+{
+    debug_me "log_maintenance"
+    find "$(dirname "${SCRIPTLOG}")" -mindepth 1 -maxdepth 1 -name "*$(basename "$0" .sh)_script_*.log" -mtime +${PURGE_LOG_TIME} -delete
+    find "$(dirname "${SCRIPTLOG}")" -mindepth 1 -maxdepth 1 -name "*$(basename "$0" .sh)_script_*.err" -mtime +${PURGE_LOG_TIME} -delete
+}
+
+
+########################################################################
+#
+# / FUNCTIONS
+#
+########################################################################
+
+########################################################################
+#
+# MAIN
+#
+########################################################################
+
+
+
+no_colors_output
+
+[[ ! -d $(dirname "${SCRIPTLOG}") ]] && mkdir -p "$(dirname "${SCRIPTLOG}")"
+[[ ! -d $(dirname "${SCRIPTLOGERR}") ]] && mkdir -p "$(dirname "${SCRIPTLOGERR}")"
+
+# DETECTING if the script is run by cron
+if [[ "$(tty)" = "not a tty" ]] ; then
+    set -x
+    exec > "${SCRIPTLOG}"
+    exec 2> "${SCRIPTLOGERR}"
+fi
+
+if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
+    echo -e "${BLUE}DEBUGMODE${RESET} is on"
+    echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}"
+    echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}"
+    set -x
+    #exec 2> ${SCRIPTLOGERR}
+    exec 2> >(tee "${SCRIPTLOGERR}" >&2)
+    exec > >(tee "${SCRIPTLOG}" >&1)
+fi
+
+
+# just in case the script need arguments
+if [[ ! "$*" ]] ; then
+    usage
+    exit 0
+fi
+
+
+while getopts "o:hDF" arg; do
+  case $arg in
+    o)
+        OPTION=${OPTARG}
+        ;;
+    h)
+        usage
+        ;;
+    F)
+        MAYTHEFORCEBEWITHYOU=true
+        echo -e "${LIGHTBLUE}The force is strong in you?${RESET}"
+        ;;
+    D)
+        DEBUG=0
+        ;;
+    *)
+        usage
+        ;;
+  esac
+done
+
+
+
+[[ ! -f ${CONFIGFILE} ]] && abort_message "CONFIGFILE ${CONFIGFILE} NOT FOUND"
+. "${CONFIGFILE}"
+
+#[[ ${DEBUG} -eq 0 ]] && DEBUGME="bash -x"
+
+log_maintenance
+
+exit ${EXITCODE}
+
+########################################################################
+#
+# / MAIN
+#
+########################################################################
+
+

get_all_pgmetadata.sh

@@ -78,17 +78,36 @@ RESET="\033[0;00m"
 #
 ########################################################################
 
+printmsg()
+{
+   echo -e "$*"
+}
 
+abort_message()
+{
+    local ABORTCODE=1
+    [[ ${EXITCODE} -ne 0 ]] && ABORTCODE=${EXITCODE}
+    printmsg "${LIGHTRED}ERROR${RESET}: $*"
+    exit ${ABORTCODE}
+}
 
+# debug_me uses variable ${DEBUG}
 debug_me()
 {
-    if [[ ${DEBUG} -eq 0 ]] ; then 
+    if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
-        [[ "$*" ]] && echo -e "$*"
+        printmsg "${LIGHTBLUE}[DEBUG]${RESET}: $*"
-        echo -e "${BLUE}DEBUGMODE${RESET} Press enter to continue"
-        read -t ${WAITSEC}
     fi
 }
 
+warning_message()
+{
+    printmsg "${LIGHTYELLOW}[WARNING]${RESET}: $*"
+}
+
+ok_message()
+{
+    printmsg "${LIGHTGREEN}[OK]${RESET}: $*"
+}
 
 usage()
 {
@@ -100,14 +119,42 @@ usage()
     # VERY INITIAL CHECKS
 }
 
+nice_countdown()
+{
+    local WAIT_TIME=$1
+    local DEFAULT_WAIT_TIME=10
+    if [[ ! ${WAIT_TIME} ]] ; then
+        WAIT_TIME=${DEFAULT_WAIT_TIME}
+    fi
+    local WARNING=$((WAIT_TIME*2/3))
+    local CRITICAL=$((WAIT_TIME/3))
+    for ((x=WAIT_TIME; x>0; x--)) ; do
+        if [[ ${x} -gt ${WARNING} ]] ; then
+            MSG_COLOR=${LIGHTGREEN}
+        elif [[ ${x} -gt ${CRITICAL} ]] ; then
+            MSG_COLOR=${LIGHTYELLOW}
+        else
+            MSG_COLOR=${LIGHTRED}
+        fi
+        printf "\r${MSG_COLOR}%s${RESET} .." "${x}"
+        #sleep 1
+        read -r -t1
+    done
+    echo
+}
+
+
 get_dbname()
 {
-    DATABASES="$(echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1');" | ${PSQL} -q | egrep -v "^($|\(.*\)$|\-{1,}$)|datname"| awk '{print $1}')"
+    DATABASES="$(echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1', 'rdsadmin') order by 1;" | ${PSQL} -q | grep -Ev "^($|\(.*\)$|-{1,}$)|datname"| awk '{print $1}')"
+    #echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1');" | ${PSQL} -q | grep -Ev "^($|\(.*\)$|-{1,}$)|datname"| awk '{print $1}'
+    debug_me "DATABASES=${DATABASES}"
 }
 
 
 generate_full_metadata()
 {
+    #debug_me "${PGDUMP} -s ${DBNAME} > ${FULLMETADATA}"
     ${PGDUMP} -s ${DBNAME} > ${FULLMETADATA}
 }
 
@@ -131,20 +178,28 @@ generate_split_metadata()
                     SCHEMA="postgres"
                 fi
                 OUTPUTDIR="${BACKUPDIR}/${DBNAME}/${SCHEMA// /_}/${TYPE// /_}"
+                #debug_me "OUTPUTDIR=${OUTPUTDIR}"
                 # PATCH for very long file names
-                OUTPUTFILE="${OBJECT::240}"
+                if [[ ${#OBJECT} -gt 240 ]] ; then
-                OUTPUTFILE="${OUTPUTFILE// /_}"
+                    OUTPUTFILE="${OBJECT::240}"
-                if [[ -f ${OUTPUTDIR}/${OUTPUTFILE}.sql ]] ; then
+                else 
-                    for ((x=1; x<1000 ;x++)) ; do
+                    OUTPUTFILE="${OBJECT}"
-                        if [[ ! -f ${OUTPUTDIR}/${OUTPUTFILE}_${x}.sql ]] ; then 
-                            OUTPUTFILE="${OUTPUTFILE}_${x}"
-                        fi
-                    done
                 fi
+                OUTPUTFILE="${OUTPUTFILE// /_}"
+
+                #if [[ -f "${OUTPUTDIR}/${OUTPUTFILE}.sql" ]] ; then
+                #    for ((x=1; x<20 ;x++)) ; do
+                #        if [[ ! -f ${OUTPUTDIR}/${OUTPUTFILE}_${x}.sql ]] ; then 
+                #            OUTPUTFILE="${OUTPUTFILE}_${x}"
+                #            break
+                #        fi
+                #    done
+                #fi
                 OUTPUTFILE="${OUTPUTFILE}.sql"
+                #debug_me "OUTPUTFILE=${OUTPUTFILE}"
                 
                 mkdir -p "${OUTPUTDIR}"
-                echo -e "${OUTPUTDIR}/${OUTPUTFILE}"
+                #echo -e "${OUTPUTDIR}/${OUTPUTFILE}"
                 echo -e "${LINE}" > "${OUTPUTDIR}/${OUTPUTFILE}"
             fi
         fi
@@ -160,11 +215,10 @@ generate_split_metadata()
 
 dump_all_roles()
 {
-    debug_me
     if [[ ${DUMP_ROLE_PASSWORDS} -eq 0 ]] ; then
-        ${PGDUMPALL} --roles-only  > ${OUTPUTDIR}/dump_roles.sql
+        ${PGDUMPALL} --roles-only  > ${BACKUPDIR}/dump_roles.sql
     else
-        ${PGDUMPALL} --roles-only --no-role-passwords > ${OUTPUTDIR}/dump_roles.sql
+        ${PGDUMPALL} --roles-only --no-role-passwords > ${BACKUPDIR}/dump_roles.sql
     fi
 }
 
@@ -176,22 +230,15 @@ init_staff()
 
 upload_to_git()
 {
-    [[ ${DEBUG} -eq 0 ]] && set -x
+    #[[ ${DEBUG} -eq 0 ]] && set -x
 
-    local let RES=0
+    local RES
+    RES=0
 
-    debug_me
+    debug_me "upload_to_git()"
 
     cd ${BACKUPDIR}
 
-	GITVERSION=$(git --version | awk '{print $3}')
-	if [[ ${GITVERSION} =~ ^1.*$ ]] ; then
-	    NOEDIT=""
-	else
-	    NOEDIT="--no-edit"
-	fi
-	
-	
     # git version patch
     git pull ${NOEDIT} -q -f
     git add *
@@ -200,8 +247,6 @@ upload_to_git()
     # Updating repo
     [[ ${RES} -eq 0 ]] && git push && RES=$?
 
-    debug_me
-
     return ${RES}
 }
 
@@ -222,40 +267,42 @@ upload_to_git()
 [[ ! -f ${CONFIGFILE} ]] && echo -e "${LIGHTRED} CONFIGFILE ${CONFIGFILE} NOT FOUND${RESET}" && exit 1
 . ${CONFIGFILE}
 
-# DETECTING if the script is run by cron
+## DETECTING if the script is run by cron
-if [[ "$(tty)" = "not a tty" ]] ; then
+#if [[ "$(tty)" = "not a tty" ]] ; then
-	[[ ${DEBUG} -eq 0 ]] && set -x
+#	[[ ${DEBUG} -eq 0 ]] && set -x
-	exec > ${SCRIPTLOG}
+#	exec > ${SCRIPTLOG}
-	exec 2> ${SCRIPTLOGERR}
+#	exec 2> ${SCRIPTLOGERR}
-elif [[ ${DEBUG} -eq 0 ]] ; then
+#elif [[ ${DEBUG} -eq 0 ]] ; then
-    echo -e "${BLUE}DEBUGMODE${RESET} is on"
+#    echo -e "${BLUE}DEBUGMODE${RESET} is on"
-    #echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}"
+#    #echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}"
-    #echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}"
+#    #echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}"
-    #set -x
+#    #set -x
-	#exec > ${SCRIPTLOG}
+#	#exec > ${SCRIPTLOG}
-	#exec 2> ${SCRIPTLOGERR}
+#	#exec 2> ${SCRIPTLOGERR}
-fi
+#fi
 
 [[ ${DEBUG} -eq 0 ]] && DEBUGME="bash -x"
 
+debug_me "init_staff"
 init_staff
-debug_me
+debug_me  "get_dbname"
 get_dbname
-debug_me
+debug_me "main loop over dbs"
 # multi-database support (auto mode)
 for DBNAME in ${DATABASES} ; do
     FULLMETADATA="${BACKUPDIR}/FULL_${DBNAME}_METADATA.sql"
 
-    debug_me "generate_full_metadata"
+    debug_me "Database=${YELLOW}${DBNAME}${RESET}"
     generate_full_metadata
     if [[ ${SPLIPT_METADATA} -eq 0 ]] ; then
-        debug_me "generate_split_metadata"
         generate_split_metadata
     fi
-    debug_me "dump_all_roles"
+    #nice_countdown 6
-    dump_all_roles
 done
 
+#debug_me "dump_all_roles"
+#dump_all_roles
+
 if [[ ${ISGITREPO} -eq 0 ]]  ; then
     debug_me "upload_to_git"
 	upload_to_git