Compare commits

..

1 Commits

Author SHA1 Message Date
dodger 73c24e9301 Initial commit 2022-07-26 17:00:31 +02:00
6 changed files with 1 additions and 680 deletions
-2
View File
@@ -1,2 +0,0 @@
*config
metadata/*
-1
View File
@@ -1 +0,0 @@
disable=SC2086,SC2046,SC2034,SC2162,SC2196,SC2317,SC2164,SC2316,SC1090,SC2035
+1 -1
View File
@@ -1,3 +1,3 @@
# get_all_pgmetadata
Get all metadata from a postgresql database
get_all_pgmetadata
-288
View File
@@ -1,288 +0,0 @@
#!/bin/bash
# Exit codes:
# 1 :
# 2 :
# 3 :
# 4 :
########################################################################
# INIT
########################################################################
CONFIGFILE="$(dirname "$(readlink -f "$0")")/$(basename "$(readlink -f "$0")" .sh).config"
########################################################################
#/INIT
########################################################################
########################################################################
#
# CONSTANTS
#
########################################################################
# colors
BOLD="\e[1m"
GREEN="\e[32m"
LIGHTGREEN="${BOLD}${GREEN}"
#RED="\033[1;31m"
LIGHTRED="\033[1;31m"
BLUE="\e[34m"
LIGHTBLUE="${BOLD}${BLUE}"
YELLOW="\e[33m"
LIGHTYELLOW="${BOLD}${YELLOW}"
#WHITE="\033[0;37m"
RESET="\033[0;00m"
NOW="$(date +%Y%m%d%H%M%S)"
# set to 0 to output messages without colors
NOCOLOR=0
# set to 0 for debugging
DEBUG=0
#DEBUG=1
EXITCODE=0
########################################################################
#
# / CONSTANTS
#
########################################################################
########################################################################
#
# VARIABLES
#
########################################################################
SCRIPTLOG="$(dirname "$(readlink -f "$0")")/logs/$(basename "$0" .sh)_script_${NOW}.log"
SCRIPTLOGERR="$(dirname "$(readlink -f "$0")")/logs/$(basename "$0" .sh)_script_${NOW}.err"
# delete logs older than PURGE_LOG_TIME days
PURGE_LOG_TIME=30
########################################################################
#
# / VARIABLES
#
########################################################################
########################################################################
#
# FUNCTIONS
#
########################################################################
usage()
{
printf "%s${LIGHTRED}USAGE:${RESET}
$0 -u USERNAME -t TEMPLAGE_FILE [-h] [-D]
-u USERNAME ${LIGHTGREEN}USERNAME${RESET} to be created
-t TEMPLATE_FILE ${LIGHTGREEN}TEMPLATE_FILE${RESET} to be used (must exist)
-p PASSWORD ${LIGHTGREEN}PASSWORD${RESET} for the user
-h this help
-D DEBUG mode
"
# VERY INITIAL CHECKS
}
no_colors_output()
{
if [[ ${NOCOLOR} -eq 0 ]] ; then
# colors
BOLD=""
GREEN=""
LIGHTGREEN=""
LIGHTRED=""
BLUE=""
LIGHTBLUE=""
YELLOW=""
LIGHTYELLOW=""
RESET=""
fi
}
printmsg()
{
echo -e "$*"
}
output_log()
{
if [[ "${QUIETOUTPUT}" == true ]]; then
printmsg "$*" >> "${OUTPUTFILE}"
else
printmsg "$*" | tee -a "${OUTPUTFILE}"
fi
}
abort_message()
{
local ABORTCODE=1
[[ ${EXITCODE} -ne 0 ]] && ABORTCODE=${EXITCODE}
printmsg "${LIGHTRED}ERROR${RESET}: $*"
exit ${ABORTCODE}
}
# debug_me uses variable ${DEBUG}
debug_me()
{
if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
printmsg "${LIGHTBLUE}[DEBUG]${RESET}: $*"
fi
}
warning_message()
{
printmsg "${LIGHTYELLOW}[WARNING]${RESET}: $*"
}
ok_message()
{
printmsg "${LIGHTGREEN}[OK]${RESET}: $*"
}
nice_countdown()
{
local WAIT_TIME=$1
local DEFAULT_WAIT_TIME=10
if [[ ! ${WAIT_TIME} ]] ; then
WAIT_TIME=${DEFAULT_WAIT_TIME}
fi
local WARNING=$((WAIT_TIME*2/3))
local CRITICAL=$((WAIT_TIME/3))
for ((x=WAIT_TIME; x>0; x--)) ; do
if [[ ${x} -gt ${WARNING} ]] ; then
MSG_COLOR=${LIGHTGREEN}
elif [[ ${x} -gt ${CRITICAL} ]] ; then
MSG_COLOR=${LIGHTYELLOW}
else
MSG_COLOR=${LIGHTRED}
fi
printf "\r${MSG_COLOR}%s${RESET} .." "${x}"
sleep 1
done
echo
}
# ssh_it uses variable ${DEBUG}
ssh_it()
{
local RESSULT=99
if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
${SSHIT} "$*"
RESSULT=$?
else
${SSHIT} "$*" 2>/dev/null
RESSULT=$?
fi
return "${RESSULT}"
}
log_maintenance()
{
debug_me "log_maintenance"
find "$(dirname "${SCRIPTLOG}")" -mindepth 1 -maxdepth 1 -name "*$(basename "$0" .sh)_script_*.log" -mtime +${PURGE_LOG_TIME} -delete
find "$(dirname "${SCRIPTLOG}")" -mindepth 1 -maxdepth 1 -name "*$(basename "$0" .sh)_script_*.err" -mtime +${PURGE_LOG_TIME} -delete
}
########################################################################
#
# / FUNCTIONS
#
########################################################################
########################################################################
#
# MAIN
#
########################################################################
no_colors_output
[[ ! -d $(dirname "${SCRIPTLOG}") ]] && mkdir -p "$(dirname "${SCRIPTLOG}")"
[[ ! -d $(dirname "${SCRIPTLOGERR}") ]] && mkdir -p "$(dirname "${SCRIPTLOGERR}")"
# DETECTING if the script is run by cron
if [[ "$(tty)" = "not a tty" ]] ; then
set -x
exec > "${SCRIPTLOG}"
exec 2> "${SCRIPTLOGERR}"
fi
if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
echo -e "${BLUE}DEBUGMODE${RESET} is on"
echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}"
echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}"
set -x
#exec 2> ${SCRIPTLOGERR}
exec 2> >(tee "${SCRIPTLOGERR}" >&2)
exec > >(tee "${SCRIPTLOG}" >&1)
fi
# just in case the script need arguments
if [[ ! "$*" ]] ; then
usage
exit 0
fi
while getopts "o:hDF" arg; do
case $arg in
o)
OPTION=${OPTARG}
;;
h)
usage
;;
F)
MAYTHEFORCEBEWITHYOU=true
echo -e "${LIGHTBLUE}The force is strong in you?${RESET}"
;;
D)
DEBUG=0
;;
*)
usage
;;
esac
done
[[ ! -f ${CONFIGFILE} ]] && abort_message "CONFIGFILE ${CONFIGFILE} NOT FOUND"
. "${CONFIGFILE}"
#[[ ${DEBUG} -eq 0 ]] && DEBUGME="bash -x"
log_maintenance
exit ${EXITCODE}
########################################################################
#
# / MAIN
#
########################################################################
-70
View File
@@ -1,70 +0,0 @@
#!/bin/bash
####################################
# cron vars
####################################
PGDATA=/var/lib/pgsql/12/data
PGDUMP="$(which pg_dump)"
PGDUMPALL="$(which pg_dumpall)"
PGDUMP="${PGDUMP} -h 127.0.0.1"
PGDUMPALL="${PGDUMPALL} -h 127.0.0.1"
PSQL="$(which psql) -h 127.0.0.1"
TODAY="$(date +%Y%m%d)"
NOW="$(date +%Y%m%d%H%M%S)"
####################################
# output folders
####################################
BACKUPDIR="/var/lib/postgres/scripts/get_all_pgmetadata/${HOSTNAME,,}"
LOGDIR="${BACKUPDIR}/logs/"
# multi-db support, this is auto
#FULLMETADATA="${BACKUPDIR}/FULL_${DBNAME}_METADATA.sql"
# multi-db support, this is auto
#DIFFILE="${BACKUPDIR}/FULL_${DBNAME}_METADATA.sql"
LOGFILE=${LOGDIR}/$(basename $0 .sh)_${NOW}.log
SCRIPTLOG="${LOGFILE}"
ERRFILE=${LOGDIR}/$(basename $0 .sh)_${NOW}.err
SCRIPTLOGERR="${ERRFILE}"
####################################
# output options
####################################
# set to 0 to split full metadata into files (DEFAULT behavior)
SPLIT_METADATA=0
# dump user passwords
DUMP_ROLE_PASSWORDS=0
####################################
# DEBUG
####################################
# set to 0 to debug
DEBUG=0
DEBUG=1
# For debugging
WAITSEC=10
####################################
# git usage
####################################
# for git usage you'll have to create a git repo on the git server
# then import it into the ${BACKUPDIR} so authentication & config is done
# then this script will make `git push` automatically if ISGITREPO=0
# set to 0 if git is in place
ISGITREPO=0
ISGITREPO=1
-318
View File
@@ -1,318 +0,0 @@
#!/bin/bash
# Exit codes:
# 1 :
# 2 :
# 3 :
# 4 :
########################################################################
# INIT
########################################################################
# moved to config
#DEBUGME="${1:2}"
#if [[ "${DEBUGME,,}" = "debug" ]] ; then
# DEBUG=0
#else
# DEBUG=1
#fi
CONFIGFILE="$(dirname $(readlink -f $0))/$(basename $(readlink -f $0) .sh).config"
########################################################################
#/INIT
########################################################################
########################################################################
#
# CONSTANTS
#
########################################################################
# colors
BOLD="\e[1m"
GREEN="\e[32m"
LIGHTGREEN="${BOLD}${GREEN}"
RED="\033[1;31m"
LIGHTRED="\033[1;31m"
BLUE="\e[34m"
LIGHTBLUE="${BOLD}${BLUE}"
YELLOW="\e[33m"
LIGHTYELLOW="${BOLD}${YELLOW}"
WHITE="\033[0;37m"
RESET="\033[0;00m"
########################################################################
#
# / CONSTANTS
#
########################################################################
########################################################################
#
# VARIABLES
#
########################################################################
########################################################################
#
# / VARIABLES
#
########################################################################
########################################################################
#
# FUNCTIONS
#
########################################################################
printmsg()
{
echo -e "$*"
}
abort_message()
{
local ABORTCODE=1
[[ ${EXITCODE} -ne 0 ]] && ABORTCODE=${EXITCODE}
printmsg "${LIGHTRED}ERROR${RESET}: $*"
exit ${ABORTCODE}
}
# debug_me uses variable ${DEBUG}
debug_me()
{
if [[ "${DEBUG}" && ${DEBUG} -eq 0 ]] ; then
printmsg "${LIGHTBLUE}[DEBUG]${RESET}: $*"
fi
}
warning_message()
{
printmsg "${LIGHTYELLOW}[WARNING]${RESET}: $*"
}
ok_message()
{
printmsg "${LIGHTGREEN}[OK]${RESET}: $*"
}
usage()
{
printf "%s${LIGHTRED}USAGE:${RESET}
$0 VAR1 VAR2
Where VAR1 allowed: ${LIGHTGREEN}BLA${RESET}
And VAR2 allowed: ${LIGHTGREEN}$BLA${RESET}\n"
# VERY INITIAL CHECKS
}
nice_countdown()
{
local WAIT_TIME=$1
local DEFAULT_WAIT_TIME=10
if [[ ! ${WAIT_TIME} ]] ; then
WAIT_TIME=${DEFAULT_WAIT_TIME}
fi
local WARNING=$((WAIT_TIME*2/3))
local CRITICAL=$((WAIT_TIME/3))
for ((x=WAIT_TIME; x>0; x--)) ; do
if [[ ${x} -gt ${WARNING} ]] ; then
MSG_COLOR=${LIGHTGREEN}
elif [[ ${x} -gt ${CRITICAL} ]] ; then
MSG_COLOR=${LIGHTYELLOW}
else
MSG_COLOR=${LIGHTRED}
fi
printf "\r${MSG_COLOR}%s${RESET} .." "${x}"
#sleep 1
read -r -t1
done
echo
}
get_dbname()
{
DATABASES="$(echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1', 'rdsadmin') order by 1;" | ${PSQL} -q | grep -Ev "^($|\(.*\)$|-{1,}$)|datname"| awk '{print $1}')"
#echo "select datname from pg_database where datname not in ('postgres', 'template0', 'template1');" | ${PSQL} -q | grep -Ev "^($|\(.*\)$|-{1,}$)|datname"| awk '{print $1}'
debug_me "DATABASES=${DATABASES}"
}
generate_full_metadata()
{
#debug_me "${PGDUMP} -s ${DBNAME} > ${FULLMETADATA}"
${PGDUMP} -s ${DBNAME} > ${FULLMETADATA}
}
generate_split_metadata()
{
local OBJECT=""
local SCHEMA=""
local TYPE=""
local OUTPUTDIR=""
local OUTPUTFILE=""
# reading from &3 to allow debug mode to stop the loop
while IFS='' read LINE <&3 ; do
if [[ "${LINE}" =~ ^--\ Name:\ (.*)\;\ Type:\ (.*)\;\ Schema:\ (.*)\;\ Owner:\ (.*) ]] ; then
OBJECT="${BASH_REMATCH[1]}"
SCHEMA="${BASH_REMATCH[3]}"
TYPE="${BASH_REMATCH[2]}"
#debug_me "${LINE}"
if [[ "${OBJECT}" && "${SCHEMA}" ]] ; then
if [[ "${SCHEMA}" = "-" ]] ; then
SCHEMA="postgres"
fi
OUTPUTDIR="${BACKUPDIR}/${DBNAME}/${SCHEMA// /_}/${TYPE// /_}"
#debug_me "OUTPUTDIR=${OUTPUTDIR}"
# PATCH for very long file names
if [[ ${#OBJECT} -gt 240 ]] ; then
OUTPUTFILE="${OBJECT::240}"
else
OUTPUTFILE="${OBJECT}"
fi
OUTPUTFILE="${OUTPUTFILE// /_}"
#if [[ -f "${OUTPUTDIR}/${OUTPUTFILE}.sql" ]] ; then
# for ((x=1; x<20 ;x++)) ; do
# if [[ ! -f ${OUTPUTDIR}/${OUTPUTFILE}_${x}.sql ]] ; then
# OUTPUTFILE="${OUTPUTFILE}_${x}"
# break
# fi
# done
#fi
OUTPUTFILE="${OUTPUTFILE}.sql"
#debug_me "OUTPUTFILE=${OUTPUTFILE}"
mkdir -p "${OUTPUTDIR}"
#echo -e "${OUTPUTDIR}/${OUTPUTFILE}"
echo -e "${LINE}" > "${OUTPUTDIR}/${OUTPUTFILE}"
fi
fi
if [[ -f "${OUTPUTDIR}/${OUTPUTFILE}" ]] ; then
if [[ ! "${LINE}" =~ ^--.*$ ]] ; then
# echo "Skipping line comment"
# else
echo -e "${LINE}" >> "${OUTPUTDIR}/${OUTPUTFILE}"
fi
fi
done 3< ${FULLMETADATA}
}
dump_all_roles()
{
if [[ ${DUMP_ROLE_PASSWORDS} -eq 0 ]] ; then
${PGDUMPALL} --roles-only > ${BACKUPDIR}/dump_roles.sql
else
${PGDUMPALL} --roles-only --no-role-passwords > ${BACKUPDIR}/dump_roles.sql
fi
}
init_staff()
{
[[ ! -d ${BACKUPDIR} ]] && mkdir -p ${BACKUPDIR}
cd ${BACKUPDIR}
}
upload_to_git()
{
#[[ ${DEBUG} -eq 0 ]] && set -x
local RES
RES=0
debug_me "upload_to_git()"
cd ${BACKUPDIR}
# git version patch
git pull ${NOEDIT} -q -f
git add *
git commit -a -m "$(date) release"
RES=$?
# Updating repo
[[ ${RES} -eq 0 ]] && git push && RES=$?
return ${RES}
}
########################################################################
#
# / FUNCTIONS
#
########################################################################
########################################################################
#
# MAIN
#
########################################################################
[[ ! -f ${CONFIGFILE} ]] && echo -e "${LIGHTRED} CONFIGFILE ${CONFIGFILE} NOT FOUND${RESET}" && exit 1
. ${CONFIGFILE}
## DETECTING if the script is run by cron
#if [[ "$(tty)" = "not a tty" ]] ; then
# [[ ${DEBUG} -eq 0 ]] && set -x
# exec > ${SCRIPTLOG}
# exec 2> ${SCRIPTLOGERR}
#elif [[ ${DEBUG} -eq 0 ]] ; then
# echo -e "${BLUE}DEBUGMODE${RESET} is on"
# #echo -e "\t SCRIPTLOG will be ${SCRIPTLOG}"
# #echo -e "\t SCRIPTLOGERR will be ${SCRIPTLOGERR}"
# #set -x
# #exec > ${SCRIPTLOG}
# #exec 2> ${SCRIPTLOGERR}
#fi
[[ ${DEBUG} -eq 0 ]] && DEBUGME="bash -x"
debug_me "init_staff"
init_staff
debug_me "get_dbname"
get_dbname
debug_me "main loop over dbs"
# multi-database support (auto mode)
for DBNAME in ${DATABASES} ; do
FULLMETADATA="${BACKUPDIR}/FULL_${DBNAME}_METADATA.sql"
debug_me "Database=${YELLOW}${DBNAME}${RESET}"
generate_full_metadata
if [[ ${SPLIPT_METADATA} -eq 0 ]] ; then
generate_split_metadata
fi
#nice_countdown 6
done
#debug_me "dump_all_roles"
#dump_all_roles
if [[ ${ISGITREPO} -eq 0 ]] ; then
debug_me "upload_to_git"
upload_to_git
fi
exit ${EXITCODE}
########################################################################
#
# / MAIN
#
########################################################################