#!/bin/bash
#############################################################
# Name:        Supportconfig Plugin for SUSE SAP 
# Description: Gathers important troubleshooting information
#              about a SUSE SAP HANA/NetWeaver configurations
# License:     GPLv2
#############################################################

SVER='1.0.6'
RCFILE="/usr/lib/supportconfig/resources/supportconfig.rc"
TITLE="SUSE SAP"
LF=plugin-ha_sap.txt
INSTANCES=()

[ -s $RCFILE ] && . $RCFILE || { echo "ERROR: Initializing resource file: $RCFILE"; exit 1; }

validate_rpm_if_installed() {
	THISRPM=$1
	echo "#==[ Validating RPM ]=================================#"
	if rpm -q $THISRPM >/dev/null 2>&1; then
		echo "# rpm -V $THISRPM"
		if rpm -V $THISRPM; then
			echo "Status: Passed"
		else
			echo "Status: WARNING"
		fi
	else
		echo "package $THISRPM is not installed"
		echo "Status: Skipped"
	fi
	echo
}

collect_instance_log() {
	local LFILE
	for LFILE in "${@}" ; do
		echo "#==[ instance Log File '$LFILE' ]=============#"
		echo "# ${LFILE}"
		cat "${LFILE}"
		echo "######"
	done
}

## Docs  ##
log_entry $LF section "Referencee Documentation"
echo -e "Current Documentation:
\t https://documentation.suse.com/en-us/sles-sap/15-SP6/
\t https://documentation.suse.com/en-us/sles-sap/15-SP5/
\t https://documentation.suse.com/en-us/sles-sap/15-SP4/
\t https://documentation.suse.com/en-us/sles-sap/15-SP3/
\t https://documentation.suse.com/en-us/sles-sap/15-SP2/
\t https://documentation.suse.com/en-us/sles-sap/15-SP1/

Tuning: 
\t https:://documentation.suse.com/sles-sap/15-SP5/single-html/SLES-SAP-guide/index.html#cha-tune

Best Practice Guides: \t https://documentation.suse.com/sbp/sap/
\tSAP HANA System Replication Scale-Out - Performance-Optimized Scenario:\t https://documentation.suse.com/sbp/sap-15/html/SLES4SAP-hana-angi-scaleout-perfopt-15/index.html
\tHow to upgrade to SAPHanaSR-angi:\t https://www.suse.com/c/how-to-upgrade-to-saphanasr-angi/
\tSAP HANA SR Performance Optimized Scale up:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-sr-guide-PerfOpt-15_en.pdf
\tSAP HANA SR Performance Optimized Scale out:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-scaleOut-PerfOpt-15_en.pdf
\tSAP HANA SR Performance Optimized Scale out Multi-Target:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-scaleout-multitarget-perfopt-15_en.pdf
\tSAP HANA SR Cost Optimized Scale Up:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-sr-guide-costopt-15_en.pdf
\tENSA1 SAP NetWeaver Enqueue Replication:\t https://documentation.suse.com/sbp/sap-15/pdf/SAP-nw740-sle15-setupguide_en.pdf
\tENSA2 SAP S/4 HANA - Enqueue Replication:\t https://documentation.suse.com/sbp/sap-15/pdf/SAP-S4HA10-setupguide-simplemount-sle15_en.pdf
"
#############################################################
log_entry $LF section "Supportconfig Plugin for $TITLE, v${SVER}"
RPMLIST="
SAPHanaSR-angi
SAPHanaSR
sapconf
tuned
saptune
sap-suse-cluster-connector
sap_suse_cluster_connector
SAPHanaSR-ScaleOut
resource-agents
sapstartsrv-resource-agents
"

for THISRPM in $RPMLIST; do
	validate_rpm_if_installed $THISRPM
done
## Authentication ## 
log_entry $LF section "Authentication information"
	log_cmd $LF "grep -v '^\s*\#\|^$' /etc/nsswitch.conf | head -5"
	log_cmd $LF "grep -E '^[[:alnum:]]{3}adm:' /etc/passwd"
	log_cmd $LF "grep -E '^sap' /etc/passwd"
	log_cmd $LF "grep -E '^[[:alnum:]]{3}adm:' /etc/group"
	log_cmd $LF "grep -E -e '^sap' -e '^sdba:' -e '^dba:' /etc/group"
	log_cmd $LF "grep -E -R -e '[[:alnum:]]{3}adm' -e 'crm_attribute' -e 'SAPHanaSR' /etc/sudoers /etc/sudoers.d/"

## SAP BASIC Tuning ##
log_entry $LF section "SAP Tuning information"

RELEASE=$(grep ^VERSION= /etc/os-release)
echo -e  "OS Release: $RELEASE \n"
if [ $RELEASE == 'VERSION="11.4"' ]; then
	log_cmd $LF "chkconfig -l boot.sapconf"
	log_cmd $LF "cat /etc/sysconfig/sapconf | grep -v '^\s*\#\|^$'"
else 
	log_cmd $LF "systemctl status sapconf"
	log_cmd $LF "systemctl status tuned.service"
	if [ $? -eq 0 ]
	then
		noSaptuneProfile="true"
		log_cmd $LF "tuned-adm active"
		if [ $? -eq 0 ]; then
			profile=$(cat /etc/tuned/active_profile)
			if [ "$profile" == "saptune" ]; then
				log_cmd $LF "saptune version"
				log_cmd $LF "saptune solution list"
				log_cmd $LF "saptune note list"
				log_cmd $LF "saptune solution verify"
				noSaptuneProfile=""
			fi
		fi
	else
		noTuned="true"
        fi
	log_cmd $LF "systemctl status saptune.service"
	if [ $? -eq 0 ]; then
		log_cmd $LF "saptune version"
		echo -e "HINT: since saptune 3.0 a special saptune supportconfig plugin exists\n"
		log_cmd $LF "saptune status"
		log_cmd $LF "saptune solution list"
		log_cmd $LF "saptune note list"
		log_cmd $LF "saptune solution verify"
	else
		noSaptune="true"
	fi
	if [[ "$noSaptune" == "true"  && ( "$noTuned" == "true" || "$noSaptuneProfile" == "true" ) ]]; then
		echo "WARNING: tuned.service or saptune not configured"
	fi
fi 

## SAP Host Agent ##
log_entry $LF section "SAP Host Agent Information"
log_cmd $LF "/usr/sap/hostctrl/exe/saphostexec -version |tail -30"
log_cmd $LF "/usr/sap/hostctrl/exe/saphostexec -status"
     if [ $? -eq 1 ]
     then
        echo -e "ERROR: Please make sure 'sapinit.service' is enabled and running. \n"
     fi
log_cmd $LF "systemctl status sapinit"
log_cmd $LF "/usr/bin/cat /usr/sap/sapservices"
log_cmd $LF "systemctl status saphostagent.service"
log_cmd $LF "systemctl status sapping.service"
log_cmd $LF "systemctl status sappong.service"
log_cmd $LF "systemctl list-unit-files | grep -i sap"
log_cmd $LF "systemd-cgls -u SAP.slice"

## SAP Exit Code Index ##
SAPCONTROLEXIT="sapcontrol return codes: \n 
  0  Last webmethod call successful \n 
  1  Last webmethod call failed, invalid parameter \n 
  2  StartWait, StopWait, WaitforStarted, WaitforStopped, RestartServiceWait timed out \n 
  3  GetProcessList succeeded, all processes running correctly \n 
  4  GetProcessList succeeded, all processes stopped \n"
SAPHDB1="lanscapeHostConfiguration.py return codes: \n 
  0  Fatal \n 
  1  Error \n
  2  Warning \n 
  3  Info \n 
  4  OK \n 
  5  Ignore \n"
SAPHDB2="systemReplicationStatus.py return codes: \n 
  10  NoHSR \n
  11  Error \n 
  12  Unknown ( Normal rc for Secondaray/slave node ) \n 
  13  Initializing \n 
  14  Syncing \n 
  15  Active ( Desired state for Primary/master node ) \n"

ENSAMSG1="## ENSA1 ##  (Default Standalone Enqueue Server) \n
    Service names: 'msg_server', 'enserver', and 'enrepserver' \n
    Process names: 'ms.sap*', 'er.sap*' \n"
ENSAMSG2="## ENSA2 ##  (Default starting with S/4HANA ABAP Platform 1809) \n
    Service names: 'msg_server', 'enq_server', and 'enq_replicator' \n
    Process names: 'ms.sap*', 'enq.sap*', 'enqr.sap*' \n"

log_entry $LF section "$TITLE retrieve SAP Instances"
# Get SAP Instances using saphostctrl function. Relies on sap interface to be working. 
SAP_CONFIG="sap_config.txt"
if [ -x /usr/sap/hostctrl/exe/saphostctrl ]; then
	/usr/sap/hostctrl/exe/saphostctrl -function ListInstances | awk '{print $4,$6}' &> $SAP_CONFIG

	while read LINE
	do
		SID=$(echo $LINE | awk '{print $1}')
		SIDADM="$(tr '[:upper:]' '[:lower:]' <<< $SID)adm"
		INST=$(echo $LINE | awk '{print $2}')
		INST_NAME="SID=$SID:InstanceNumber=$INST"
		INSTANCES+=("${INST_NAME} ${SIDADM} ${SID} ${INST}")
	done < $SAP_CONFIG
	rm -r $SAP_CONFIG
fi

if [ -z "$SID" ]; then
	if [ -x /usr/sap/sapservices ]; then
		# saphostctrl does not report instances, search for systemd services
		# or read from /usr/sap/sapservices
		sapservices=$(systemctl -t service list-unit-files | sed -n 's/\(^SAP.*\).service.*/\1/p')
		if [ -z "$sapservices" ]; then
			# no systemd services available, try /usr/sap/sapservices
			sapservices=$(awk '$1!~/^#/ {print $3}' /usr/sap/sapservices | awk -F= '{print $2}')
			for s in $sapservices; do
				# /usr/sap/HA1/SYS/profile/HA1_HDB10_node11
				service=${s##*/}
				SID=${service:0:3}
				SIDADM=${SID,,}adm
				INST=${service:7:2}
				INST_NAME="SID=$SID:InstanceNumber=$INST"
				INSTANCES+=("${INST_NAME} ${SIDADM} ${SID} ${INST}")
			done
		else
			# SAP naming schema for services - SAP<SID>_<INSTNO>.service
			for service in $sapservices; do
				# SAPHA1_00
				SID=${service:3:3}
				SIDADM=${SID,,}adm
				INST=${service:7:2}
				INST_NAME="SID=$SID:InstanceNumber=$INST"
				INSTANCES+=("${INST_NAME} ${SIDADM} ${SID} ${INST}")
			done
		fi
	else
		echo -e "ERROR: No SIDs found! \n"
		
	fi
fi

log_entry $LF section "$TITLE Troubleshooting Commands"
log_cmd $LF "/usr/sap/hostctrl/exe/saphostctrl -function ListInstances"
if [ -n "$SIDADM" ]; then
	log_cmd $LF "ps -U $SIDADM -f"
fi

# Run through each instance detected and display information about it
VHOST=$(hostname)
INST_CNT=1
LABEL=0
ADM=1
SID=2
NUM=3
for INSTANCE in "${INSTANCES[@]}"
do
	# take the INSTANCE variable and turn it into a new array called ELEMENT. Rather than refer to the items in the array by their numeric placement the above 4 variables will be used as reference.
	read -r -a ELEMENT <<< $INSTANCE
	log_entry $LF section "$TITLE Instance $INST_CNT of ${#INSTANCES[@]}: ${ELEMENT[${LABEL}]}"
	HDBEXIST=true
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDB version'"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'python --version'"
	log_cmd $LF "/usr/bin/id ${ELEMENT[${ADM}]}"
	if [[ $? == 0 ]]; then
		echo "RETURN CODE: $?"; echo
		log_cmd $LF "SAPHanaSR-monitor"
		echo "RETURN_CODE: $?"; echo 
		log_cmd $LF "SAPHanaSR-showAttr"
		echo "RETURN_CODE: $?"; echo 
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'hdbnsutil -sr_state'"
		echo "RETURN CODE: $?"; echo
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDBSettings.sh landscapeHostConfiguration.py --sapcontrol=1'"
		echo "RETURN CODE: $?"; echo -e $SAPHDB1; echo
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDBSettings.sh systemReplicationStatus.py'"
		echo "RETURN CODE: $?"; echo -e $SAPHDB2; echo
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDBSettings.sh systemOverview.py'"
		echo "RETURN CODE: $?"; echo
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDB info'"
		echo "RETURN CODE: $?"; echo
	else
		echo "RETURN CODE: $?"; echo
		HDBEXIST=false
	fi
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetStartProfile'"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetSystemInstanceList'"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetProcessList'"
	echo "RETURN CODE: $?"; echo -e $SAPCONTROLEXIT; echo; echo -e $ENSAMSG1; echo -e $ENSAMSG2

	INSTANCE_PROFILE_DIR=$(su - ${ELEMENT[${ADM}]} -c 'cdpro && pwd' 2>/dev/null)
	INSTANCE_CONFIG_DIR=$(su - ${ELEMENT[${ADM}]} -c 'cdcoc && pwd' 2>/dev/null)
	if [[ -d $INSTANCE_CONFIG_DIR ]] && [[ $HDBEXIST == true ]]; then
		conf_files $LF  ${INSTANCE_CONFIG_DIR}/global.ini ${INSTANCE_CONFIG_DIR}/nameserver.ini ${INSTANCE_CONFIG_DIR}/daemon.ini
	fi
	if [[ -d $INSTANCE_PROFILE_DIR ]]; then
		conf_files $LF  ${INSTANCE_PROFILE_DIR}/${ELEMENT[${SID}]}_*
	fi
	
	log_cmd $LF "systemctl status SAP${ELEMENT[${SID}]}_${ELEMENT[${NUM}]}.service"

	# SUSE HANA hook script logs
	# /usr/sap/HA1/HDB10/hanacost1/trace/nameserver_suschksrv.trc
	lfile=/usr/sap/${ELEMENT[${SID}]}/HDB${ELEMENT[${NUM}]}/${VHOST}/trace/nameserver_suschksrv.trc
	collect_instance_log ${lfile}
	# /usr/sap/HA1/HDB10/hanacost1/trace/nameserver_saphanasr_multitarget_hook.trc
	lfile=/usr/sap/${ELEMENT[${SID}]}/HDB${ELEMENT[${NUM}]}/${VHOST}/trace/nameserver_saphanasr_multitarget_hook.trc
	collect_instance_log ${lfile}

	## Adding support for sap-suse-cluster-connector  ##
        log_cmd $LF "rpm -qa | grep -E 'sap-suse-cluster-connector|sap_suse_cluster_connector'"
        if [ $? -eq 0 ]; then
                log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function HAGetFailoverConfig'"
                echo "RETURN CODE: $?"; echo 
                log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function HACheckFailoverConfig'"
                echo "RETURN CODE: $?"; echo 
                log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function HACheckConfig'"
                echo "RETURN CODE: $?"; echo 

		# collect instance logs
		# /usr/sap/EN2/ASCS00/work/sap_suse_cluster_connector.[old|log]
		lfile=/usr/sap/${ELEMENT[${SID}]}/ASCS${ELEMENT[${NUM}]}/work/sap_suse_cluster_connector.???
		collect_instance_log ${lfile}
		# /usr/sap/EN2/ASCS00/work/sapstartsrv.???
		lfile=/usr/sap/${ELEMENT[${SID}]}/ASCS${ELEMENT[${NUM}]}/work/sapstartsrv.???
		collect_instance_log ${lfile}
        fi

	 INST_CNT=$[ INST_CNT + 1 ]
done
## Adding some basic cluster commands.  Refer to ha.txt in supportconfig for additional ##
log_entry $LF section "$TITLE Basic Cluster Configuration"
	log_cmd $LF "crm_mon -A -r -1"
	log_cmd $LF "crm configure show obscure:passw*"

log_entry $LF section "$TITLE Log Files"
        log_cmd $LF "grep -E -i 'saphana|SAPDatabase|SAPInstance|SAPStartsrv|sapcontrol|saphostctrl|sap_suse_cluster_connector' /var/log/messages | tail -1000"
        if ! [ $RELEASE == 'VERSION="11.4"' ]; then
                log_cmd $LF "find /var/log/ -name pacemaker.log -exec grep -E -i 'saphana|SAPDatabase|SAPInstance|SAPStartsrv|sapcontrol|saphostctrl' {} \; | tail -1000"
        fi
