From 0457c68ccfb79125b2f6fe2c16b10ef58c6d6542 Mon Sep 17 00:00:00 2001 From: Jorge Holgado Date: Fri, 16 Aug 2024 15:54:53 +0200 Subject: [PATCH] updates on the script --- .gitignore | 1 + .shellcheckrc | 1 + get_all_pgmetadata.sh | 125 ++++++++++++++++++++++++++++-------------- 3 files changed, 85 insertions(+), 42 deletions(-) create mode 100644 .shellcheckrc diff --git a/.gitignore b/.gitignore index df30d24..f6cdce3 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *config +metadata/* diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..728bc4f --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1 @@ +disable=SC2086,SC2046,SC2034,SC2162,SC2196,SC2317,SC2164,SC2316,SC1090,SC2035 diff --git a/get_all_pgmetadata.sh b/get_all_pgmetadata.sh index 974ad97..e0df237 100755 --- a/get_all_pgmetadata.sh +++ b/get_all_pgmetadata.sh @@ -78,17 +78,36 @@ RESET="\033[0;00m" # ######################################################################## +printmsg() +{ + echo -e "$*" +} +abort_message() +{ + local ABORTCODE=1 + [[ ${EXITCODE} -ne 0 ]] && ABORTCODE=${EXITCODE} + printmsg "${LIGHTRED}ERROR${RESET}: $*" + exit ${ABORTCODE} +} +# debug_me uses variable ${DEBUG} debug_me() { - if [[ ${DEBUG} -eq 0 ]] ; then - [[ "$*" ]] && echo -e "$*" - echo -e "${BLUE}DEBUGMODE${RESET} Press enter to continue" - read -t ${WAITSEC} + if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then + printmsg "${LIGHTBLUE}[DEBUG]${RESET}: $*" fi } +warning_message() +{ + printmsg "${LIGHTYELLOW}[WARNING]${RESET}: $*" +} + +ok_message() +{ + printmsg "${LIGHTGREEN}[OK]${RESET}: $*" +} usage() { @@ -100,14 +119,42 @@ usage() # VERY INITIAL CHECKS } +nice_countdown() +{ + local WAIT_TIME=$1 + local DEFAULT_WAIT_TIME=10 + if [[ ! ${WAIT_TIME} ]] ; then + WAIT_TIME=${DEFAULT_WAIT_TIME} + fi + local WARNING=$((WAIT_TIME*2/3)) + local CRITICAL=$((WAIT_TIME/3)) + for ((x=WAIT_TIME; x>0; x--)) ; do + if [[ ${x} -gt ${WARNING} ]] ; then + MSG_COLOR=${LIGHTGREEN} + elif [[ ${x} -gt ${CRITICAL} ]] ; then + MSG_COLOR=${LIGHTYELLOW} + else + MSG_COLOR=${LIGHTRED} + fi + printf "\r${MSG_COLOR}%s${RESET} .." "${x}" + #sleep 1 + read -r -t1 + done + echo +} + + get_dbname() { - DATABASES="$(echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1');" | ${PSQL} -q | egrep -v "^($|\(.*\)$|\-{1,}$)|datname"| awk '{print $1}')" + DATABASES="$(echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1', 'rdsadmin');" | ${PSQL} -q | grep -Ev "^($|\(.*\)$|-{1,}$)|datname"| awk '{print $1}')" + #echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1');" | ${PSQL} -q | grep -Ev "^($|\(.*\)$|-{1,}$)|datname"| awk '{print $1}' + debug_me "DATABASES=${DATABASES}" } generate_full_metadata() { + #debug_me "${PGDUMP} -s ${DBNAME} > ${FULLMETADATA}" ${PGDUMP} -s ${DBNAME} > ${FULLMETADATA} } @@ -131,20 +178,22 @@ generate_split_metadata() SCHEMA="postgres" fi OUTPUTDIR="${BACKUPDIR}/${DBNAME}/${SCHEMA// /_}/${TYPE// /_}" + #debug_me "OUTPUTDIR=${OUTPUTDIR}" # PATCH for very long file names OUTPUTFILE="${OBJECT::240}" OUTPUTFILE="${OUTPUTFILE// /_}" if [[ -f ${OUTPUTDIR}/${OUTPUTFILE}.sql ]] ; then - for ((x=1; x<1000 ;x++)) ; do + for ((x=1; x<20 ;x++)) ; do if [[ ! -f ${OUTPUTDIR}/${OUTPUTFILE}_${x}.sql ]] ; then OUTPUTFILE="${OUTPUTFILE}_${x}" fi done fi OUTPUTFILE="${OUTPUTFILE}.sql" + #debug_me "OUTPUTFILE=${OUTPUTFILE}" mkdir -p "${OUTPUTDIR}" - echo -e "${OUTPUTDIR}/${OUTPUTFILE}" + #echo -e "${OUTPUTDIR}/${OUTPUTFILE}" echo -e "${LINE}" > "${OUTPUTDIR}/${OUTPUTFILE}" fi fi @@ -160,11 +209,10 @@ generate_split_metadata() dump_all_roles() { - debug_me if [[ ${DUMP_ROLE_PASSWORDS} -eq 0 ]] ; then - ${PGDUMPALL} --roles-only > ${OUTPUTDIR}/dump_roles.sql + ${PGDUMPALL} --roles-only > ${BACKUPDIR}/dump_roles.sql else - ${PGDUMPALL} --roles-only --no-role-passwords > ${OUTPUTDIR}/dump_roles.sql + ${PGDUMPALL} --roles-only --no-role-passwords > ${BACKUPDIR}/dump_roles.sql fi } @@ -176,22 +224,15 @@ init_staff() upload_to_git() { - [[ ${DEBUG} -eq 0 ]] && set -x + #[[ ${DEBUG} -eq 0 ]] && set -x - local let RES=0 + local RES + RES=0 - debug_me + debug_me "upload_to_git()" cd ${BACKUPDIR} - GITVERSION=$(git --version | awk '{print $3}') - if [[ ${GITVERSION} =~ ^1.*$ ]] ; then - NOEDIT="" - else - NOEDIT="--no-edit" - fi - - # git version patch git pull ${NOEDIT} -q -f git add * @@ -200,8 +241,6 @@ upload_to_git() # Updating repo [[ ${RES} -eq 0 ]] && git push && RES=$? - debug_me - return ${RES} } @@ -222,40 +261,42 @@ upload_to_git() [[ ! -f ${CONFIGFILE} ]] && echo -e "${LIGHTRED} CONFIGFILE ${CONFIGFILE} NOT FOUND${RESET}" && exit 1 . ${CONFIGFILE} -# DETECTING if the script is run by cron -if [[ "$(tty)" = "not a tty" ]] ; then - [[ ${DEBUG} -eq 0 ]] && set -x - exec > ${SCRIPTLOG} - exec 2> ${SCRIPTLOGERR} -elif [[ ${DEBUG} -eq 0 ]] ; then - echo -e "${BLUE}DEBUGMODE${RESET} is on" - #echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}" - #echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}" - #set -x - #exec > ${SCRIPTLOG} - #exec 2> ${SCRIPTLOGERR} -fi +## DETECTING if the script is run by cron +#if [[ "$(tty)" = "not a tty" ]] ; then +# [[ ${DEBUG} -eq 0 ]] && set -x +# exec > ${SCRIPTLOG} +# exec 2> ${SCRIPTLOGERR} +#elif [[ ${DEBUG} -eq 0 ]] ; then +# echo -e "${BLUE}DEBUGMODE${RESET} is on" +# #echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}" +# #echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}" +# #set -x +# #exec > ${SCRIPTLOG} +# #exec 2> ${SCRIPTLOGERR} +#fi [[ ${DEBUG} -eq 0 ]] && DEBUGME="bash -x" +debug_me "init_staff" init_staff -debug_me +debug_me "get_dbname" get_dbname -debug_me +debug_me "main loop over dbs" # multi-database support (auto mode) for DBNAME in ${DATABASES} ; do FULLMETADATA="${BACKUPDIR}/FULL_${DBNAME}_METADATA.sql" - debug_me "generate_full_metadata" + debug_me "Database=${YELLOW}${DBNAME}${RESET}" generate_full_metadata if [[ ${SPLIPT_METADATA} -eq 0 ]] ; then - debug_me "generate_split_metadata" generate_split_metadata fi - debug_me "dump_all_roles" - dump_all_roles + #nice_countdown 6 done +#debug_me "dump_all_roles" +#dump_all_roles + if [[ ${ISGITREPO} -eq 0 ]] ; then debug_me "upload_to_git" upload_to_git