#!/bin/bash
# prevent ugly kworker errors at end when kickstart network is detached
# and nfs root connection is severed
echo 0 > /proc/sys/kernel/hung_task_timeout_secs

#Load in any "software" to use for maintenance
echo "Welcome to the Automation Services"
echo " "
#notify that automation services started
wget -T 60 -q -o /dev/null -O /dev/null 'http://192.168.200.2/automation/updateKickstartInstallationStatus.php?kickstartInstallationId=78627&securityKey=wcyxbrvkuohajweelxgmkmwfsimhhyhf&percentComplete=0&message=Started%20Automated%20Services'

echo "Scheduled operations: "
echo " "
echo "format"
echo "hw_test"
echo " "
echo " "
NUMBER_OF_STEPS=2

#################################
#### DRIVE FORMATTING SCRIPT ####
#### ASSUMES bash shell      ####
#################################
if [ "$HOSTNAME" == "reclaimer" ]; then
echo "sanity check passed, hostname matches, proceeding with format"
else
echo "sanity check failed, hostname mismatch, aborting script"
exit;
fi
echo " "
trap "echo CTRL-C was pressed;exit" 2
CURRENT_STEP=1
CURRENT_PROGRESS=$(echo "scale=3;($CURRENT_STEP-1)/$NUMBER_OF_STEPS" | bc )

#Get list of attached drives
#DRIVES=(`lshw -short -class disk | grep '/dev/sd' | awk -F " " '{print $2}'`)
DRIVES=(`lsblk -nlp -o name,type | grep disk | awk -F" " '{print $1}'`)


echo "WARNING!  This operation is DESTRUCTIVE. "
echo "HARD DISK Drive(s) $DRIVES will be WIPED CLEAN"
echo " "
echo " This operation will start unattended in 60 seconds"
echo " You can CTRL+C to stop it "


read -t 60 -p "Enter STOP to abort this script. Any other input will start the operation!" wantsabort
echo " "

case $wantsabort in
"STOP" ) echo "Aborting";exit;;
"stop" ) echo "Aborting";exit;;
"quit" ) echo "Aborting";exit;;
"QUIT" ) echo "Aborting";exit;;
"ABORT") echo "Aborting";exit;;
"abort") echo "Aborting";exit;;
"EXIT") echo "Aborting";exit;;
"exit") echo "Aborting";exit;;
esac

echo "writing to drives in "
echo "5"
sleep 1s
echo "4"
sleep 1s
echo "3"
sleep 1s
echo "2"
sleep 1s
echo "1"
sleep 1s
echo "now"
echo " "

NUMBER_OF_DRIVES=${#DRIVES[@]}
CURRENT_DRIVE=0
LAST_PROGRESS=0
#Loop through drives to fast-wipe mbr/gpt and raid metadata
for drive in ${DRIVES[*]}; do
echo "Clearing start of drive $drive"
echo "dd if=/dev/zero of=$drive bs=512 count=1024"
dd if=/dev/zero of=$drive bs=512 count=1024
echo "mbr cleared on $drive"
echo "Clearing raid metadata on $drive"
echo "dd if=/dev/zero of=$drive bs=512 seek=$(( $(blockdev --getsz $drive) - 1024 )) count=1024"
dd if=/dev/zero of=$drive bs=512 seek=$(( $(blockdev --getsz $drive) - 1024 )) count=1024
echo "raid metadata cleared on $drive"
done

echo "Fast wipe complete, proceeding with full-disk-wipe"
wget -q -O /dev/null -o /dev/null "http://192.168.200.2/automation/updateKickstartInstallationStatus.php?kickstartInstallationId=78627&securityKey=wcyxbrvkuohajweelxgmkmwfsimhhyhf&percentComplete=0&message=MBR%20and%20Raid%20metadata%20cleared%20-%20proceeding%20with%20full%20drive%20zero%20fill" 2> /tmp/wgeterr

#Loop through drives to format
for drive in ${DRIVES[*]}; do
((++CURRENT_DRIVE))

#Get size of drive
DRIVESIZE=`blockdev --getsize64 $drive`

#Clear logfile
echo "0" > /tmp/formatProgress.log

pv -s $DRIVESIZE -n < /dev/zero > $drive 2> /tmp/formatProgress.log &
PROCESS_PID=$!
sleep 3
while kill -0 $PROCESS_PID; do
PROGRESS_ON_DRIVE=`tail -1 /tmp/formatProgress.log`
OVERALL_PROGRESS=$(echo "scale=3;($CURRENT_PROGRESS + ($PROGRESS_ON_DRIVE * ($CURRENT_DRIVE/$NUMBER_OF_DRIVES)) * ($CURRENT_STEP/$NUMBER_OF_STEPS))" | bc)
OVERALL_PROGRESS=$(echo "scale=0;$OVERALL_PROGRESS/1" | bc)
# Send progress update only when overall progress changes
if [ "$LAST_PROGRESS" -ne "$OVERALL_PROGRESS" ]; then
echo "Overall $OVERALL_PROGRESS % complete  drive $CURRENT_DRIVE of $NUMBER_OF_DRIVES  ---  (wipe of $drive - $PROGRESS_ON_DRIVE %)"
wget -q -o /dev/null -O /dev/null "http://192.168.200.2/automation/updateKickstartInstallationStatus.php?kickstartInstallationId=78627&securityKey=wcyxbrvkuohajweelxgmkmwfsimhhyhf&percentComplete=$OVERALL_PROGRESS&message=Formatting%20Drive%20$CURRENT_DRIVE%20of%20$NUMBER_OF_DRIVES%3a%20%20($drive%20%20at%20$PROGRESS_ON_DRIVE%20percent)" 2> /tmp/wgeterr
fi
LAST_PROGRESS=$OVERALL_PROGRESS
sleep 60
#	if [ "$OVERALL_PROGRESS" -gt 0 ]; then
#		echo "Test mode: ending at 1% complete"
#		killall pv 2> /dev/null
#		echo "syncing drive(s)"
#		sync
#		echo "syncing done"
#		break
#	fi
done
wait $PROCESS_PID
done
echo "performing hw tests"
SHORT_LOG_FILE=/tmp/system_hw_test_short.log
LONG_LOG_FILE=/tmp/system_hw_test_long.log

####################################
# Motherboard
####################################
MOBO_LONG_LOG=/tmp/system_info_mobo_long.log
MOBO_SHORT_LOG=/tmp/system_info_mobo_short.log
record_mobo_details() {
MOBO_MANUFACT=`dmidecode -s baseboard-manufacturer`                        #mobo manufacturer
MOBO_MODEL_NAME=`dmidecode -s baseboard-product-name`                      #model of mobo
MOBO_VERSION=`dmidecode -s baseboard-version`                              #version of mobo

dmidecode -q -t baseboard > $MOBO_LONG_LOG

echo "motherboard           manufacturer    $MOBO_MANUFACT" > $MOBO_SHORT_LOG
echo "motherboard           model   $MOBO_MODEL_NAME"      >> $MOBO_SHORT_LOG
echo "motherboard           version $MOBO_VERSION"         >> $MOBO_SHORT_LOG
}


####################################
# SYSTEM BOOT STATUS
####################################
BOOT_STATUS_LONG_LOG=/tmp/system_info_boot_long.log
BOOT_STATUS_SHORT_LOG=/tmp/system_info_boot_short.log
record_boot_details() {
BOOT_STATUS=$(dmidecode -q -t 32 | grep "Status"  | cut -d : -f2)            #boot status e.g. "No errors detected"

echo $BOOT_STATUS > $BOOT_STATUS_LONG_LOG
echo "boot          status  $BOOT_STATUS" > $BOOT_STATUS_SHORT_LOG
}


####################################
# BIOS INFORMATION
####################################
BIOS_STATUS_LONG_LOG=/tmp/system_info_bios_long.log
BIOS_STATUS_SHORT_LOG=/tmp/system_info_bios_short.log
record_bios_details() {
BIOS_VENDOR=$(dmidecode -s bios-vendor)                                    #bios maker
BIOS_VERSION=$(dmidecode -s bios-version)                                  #bios version
BIOS_RELEASE_DATE=$(dmidecode -s bios-release-date)                        #bios release date

dmidecode -q -t bios                     > $BIOS_STATUS_LONG_LOG

echo "bios          vendor  $BIOS_VENDOR"                > $BIOS_STATUS_SHORT_LOG
echo "bios          version $BIOS_VERSION"              >> $BIOS_STATUS_SHORT_LOG
echo "bios          release date    $BIOS_RELEASE_DATE" >> $BIOS_STATUS_SHORT_LOG
}

####################################
# CPU INFORMATION
####################################
CPU_LONG_LOG=/tmp/system_info_cpu_long.log
CPU_SHORT_LOG=/tmp/system_info_cpu_short.log
record_cpu_details() {
IFS=$'\r\n'                                                                #this determines what the parse tokens are
CPUS=($(dmidecode -s processor-version))
NUMBER_OF_CPUS=${#CPUS[@]}

dmidecode -t processor -q > $CPU_LONG_LOG

cat /dev/null > $CPU_SHORT_LOG
for (( c=1; c<=$NUMBER_OF_CPUS; ++c)); do
echo "processor $c      version ${CPUS[c-1]}" >> $CPU_SHORT_LOG
done
}


####################################
# MEMORY INFORMATION
####################################
RAM_LONG_LOG=/tmp/system_info_ram_long.log
RAM_SHORT_LOG=/tmp/system_info_ram_short.log
record_ram_details() {
MAX_RAM=$(dmidecode -q -t 16 | grep -i "maximum capacity" | rev | cut -d' ' -f1,2 | rev)
MAX_SLOTS=$(dmidecode -q -t 16 | grep -i "number of devices" | rev | cut -d' ' -f1 | rev)
INSTALLED_RAM=$(dmidecode -q -t 19 | grep -i "range size" | rev | cut -d' ' -f1,2 | rev)
INSTALLED_MODULES=($(dmidecode -q -t 20 | grep -i "range size" | rev | cut -d' ' -f1,2 | rev))

dmidecode -t memory -q > $RAM_LONG_LOG

echo "ram           maxcap  $MAX_RAM"        > $RAM_SHORT_LOG
echo "ram           maxslot $MAX_SLOTS"         >> $RAM_SHORT_LOG
echo "ram           installed       $INSTALLED_RAM"     >> $RAM_SHORT_LOG
for RAM in "${!INSTALLED_MODULES[@]}"; do
echo "ram       $RAM    usedslot        ${INSTALLED_MODULES[$RAM]}" >> $RAM_SHORT_LOG
done
}

####################################
# POWER SUPPLY INFO
####################################
POWER_LONG_LOG=/tmp/system_info_power_long.log
POWER_SHORT_LOG=/tmp/system_info_power_short.log
record_power_details() {
IFS=$'\r\n'                                                                #this determines what the parse tokens are
NUM_POWER_SUPPLIES=$(dmidecode -q -t 39 | grep "System Power Supply" | wc -l)

cat /dev/null > $POWER_SHORT_LOG
for (( c=1; c<=$NUM_POWER_SUPPLIES; c++)); do                              #loop through each powersupply and record it
HOT_SWAP=$(dmidecode -q -t 39 | grep "Hot Replaceable" | tail -n +$c | head -1 | rev | cut -d' ' -f1 | rev)
PLUGGED_IN=$(dmidecode -q -t 39 | grep "Plugged" | tail -n +$c | \
head -1 | rev | cut -d' ' -f1 | rev)                                                         #power supply is plugged in
STATUS=$(dmidecode -q -t 39 | grep "Status" | tail -n +$c | \
head -1 | rev | cut -d ' ' -f1 | rev)                                                         #power supply status. Probably "OK"

echo "powersupply       $c      hotswappable    $HOT_SWAP" >> $POWER_SHORT_LOG
echo "powersupply       $c      pluggedin       $PLUGGED_IN"   >> $POWER_SHORT_LOG
echo "powersupply       $c      status  $STATUS"           >> $POWER_SHORT_LOG
done

dmidecode -q -t 39 > $POWER_LONG_LOG
}

####################################
# NICS
####################################
NIC_LONG_LOG=/tmp/system_info_nic_long.log
NIC_SHORT_LOG=/tmp/system_info_nic_short.log
record_nic_details() {
IFS=$'\r\n'                                                                #this determines what the parse tokens are

NICS=($(ifconfig -a | grep "Ethernet"))
NUM_NICS=${#NICS[@]}

cat /dev/null > $NIC_SHORT_LOG
for (( c=0; c<NUM_NICS; ++c)); do
MACADDR=$(echo ${NICS[c]} |  sed -E 's/.*hwaddr (.*)/\1/i' | tr -d :)
echo "ethernet  $c      mac     $MACADDR"  >> $NIC_SHORT_LOG
done

ifconfig -a > $NIC_LONG_LOG
}

####################################
# DRIVES
####################################
#Get list of attached drives
DRIVES_LONG_LOG=/tmp/system_info_drives_long.log
DRIVES_SHORT_LOG=/tmp/system_info_drives_short.log
record_drive_details() {
DRIVES=($(lsblk -nlp -o name,type | grep disk))
NUMBER_OF_DRIVES=${#DRIVES[@]}
TEST_MAX_WAIT=0

for (( c=0; c<$NUMBER_OF_DRIVES; ++c)); do
DRIVE_NAME=$(echo ${DRIVES[c]} | awk -F" " '{print $1}')               # drive name. eg. /dev/sda
EST_WAIT=$(smartctl --test=short $DRIVE_NAME | grep "wait" | sed -E 's/.**([0-9]+).*/\1/')
if [ $? -ne 0 ]; then                                                  # sometimes smart is not enabled on the drive
echo "drives        $DRIVE_NAME     error   Unable to initialize "\
"smartctl test on $DRIVE_NAME" >> $DRIVES_SHORT_LOG
unset DRIVES[$c]
else
if [ $TEST_MAX_WAIT -lt $EST_WAIT ]; then
TEST_MAX_WAIT=$EST_WAIT
fi
fi
done

sleep ${TEST_MAX_WAIT}m

WAIT_FLAG=1
COUNT_FLAG=0
while [ $WAIT_FLAG -gt 0 ]; do
WAIT_FLAG=0
if [ $COUNT_FLAG -gt 2 ]; then
#log_error       and poop drive off of lst
break
else
for (( c=0; c<$NUMBER_OF_DRIVES; ++c)); do
DRIVE_NAME=$(echo ${DRIVES[c]} | awk -F" " '{print $1}')
PERCENT_REMAINING=$(smartctl -l selftest $DRIVE_NAME | grep -i "short offline" | head -1 | sed -E 's/.* ([0-9]+)%.*/\1/')
if [[ "$PERCENT_REMAINING" !=  "00" ]]; then
WAIT_FLAG=1
fi
done
((COUNT_FLAG++))

sleep 10
fi
done

for (( c=0; c<$NUMBER_OF_DRIVES; ++c )); do                           #begin loop through each test and get the results
DRIVE_NAME=$(echo ${DRIVES[c]} | awk -F" " '{print $1}')               # drive name. eg. /dev/sda
FAILED_TESTS=($(smartctl --attributes $DRIVE | grep -e "FAILING_NOW" -e "In_the_past"))


if [ ${#FAILED_TESTS[@]} -gt 0 ]; then                                 #record all failed tests
for FAILURE_NOTICE in FAILED_TESTS; do
echo "drives        $DRIVE_NAME     failure_notice  $FAILURE_NOTICE" \
>> $DRIVES_SHORT_LOG
done
fi


DRIVE_START_STOP_COUNT=$(smartctl --attributes $DRIVE_NAME | \
grep -i "start_stop_count" | rev | cut -d ' ' -f1 | rev)         #the last column holds the values

DRIVE_SPIN_UP_TIME=$(smartctl --attributes $DRIVE_NAME | \
grep -i "spin_up_time" | rev | cut -d ' ' -f1 | rev)

DRIVE_POWER_ON_HOURS=$(smartctl --attributes $DRIVE_NAME |  \
grep -i "power_on_hours" | rev | cut -d ' ' -f1 | rev)

DRIVE_TEMPERATURE=$(smartctl --attributes $DRIVE_NAME | \
grep -i "temperature_celsius" | rev | cut -d ' ' -f1 | rev)

DRIVE_FAMILY=$(smartctl -i $DRIVE_NAME | \
grep -i "model family:" | cut -d: -f2 | \
sed -e 's/^ *//g' -e 's/ *$//g')

DRIVE_MODEL=$(smartctl -i $DRIVE_NAME | \
grep -i "device model:" | cut -d: -f2 | \
sed -e 's/^ *//g' -e 's/ *$//g')

DRIVE_SERIAL=$(smartctl -i $DRIVE_NAME |  \
grep -i "serial number:" | cut -d: -f2 | \
sed -e 's/^ *//g' -e 's/ *$//g')

DRIVE_CAPACITY=$(smartctl -i $DRIVE_NAME | \
grep -i "user capacity:" | cut -d: -f2 | \
sed -e 's/^ *//g' -e 's/ *$//g')


echo "drives    $DRIVE_NAME     start_stop_count        $DRIVE_START_STOP_COUNT"\
>> $DRIVES_SHORT_LOG

echo "drives    $DRIVE_NAME     spin_up_time    $DRIVE_SPIN_UP_TIME"\
>> $DRIVES_SHORT_LOG

echo "drives    $DRIVE_NAME     power_on_hours  $DRIVE_POWER_ON_HOURS"\
>> $DRIVES_SHORT_LOG
echo "drives    $DRIVE_NAME     temperature_celsius     $DRIVE_TEMPERATURE"\
>> $DRIVES_SHORT_LOG

echo "drives    $DRIVE_NAME     family  $DRIVE_FAMILY" >> $DRIVES_SHORT_LOG

echo "drives    $DRIVE_NAME     model   $DRIVE_MODEL" >> $DRIVES_SHORT_LOG

echo "drives    $DRIVE_NAME     serial  $DRIVE_SERIAL" >> $DRIVES_SHORT_LOG

echo "drives    $DRIVE_NAME     capacity        $DRIVE_CAPACITY" >> $DRIVES_SHORT_LOG

smartctl -a $DRIVE_NAME >> $DRIVES_LONG_LOG
done
}


#check ipmi
#todo

#combines logs
combine_hardware_test_logs() {
cat $MOBO_SHORT_LOG         > $SHORT_LOG_FILE
cat $BOOT_STATUS_SHORT_LOG >> $SHORT_LOG_FILE
cat $BIOS_STATUS_SHORT_LOG >> $SHORT_LOG_FILE
cat $CPU_SHORT_LOG         >> $SHORT_LOG_FILE
cat $RAM_SHORT_LOG         >> $SHORT_LOG_FILE
cat $POWER_SHORT_LOG       >> $SHORT_LOG_FILE
cat $NIC_SHORT_LOG         >> $SHORT_LOG_FILE
cat $DRIVES_SHORT_LOG      >> $SHORT_LOG_FILE

cat $MOBO_LONG_LOG          > $LONG_LOG_FILE
cat $BOOT_STATUS_LONG_LOG  >> $LONG_LOG_FILE
cat $BIOS_STATUS_LONG_LOG  >> $LONG_LOG_FILE
cat $CPU_LONG_LOG          >> $LONG_LOG_FILE
cat $RAM_LONG_LOG          >> $LONG_LOG_FILE
cat $POWER_LONG_LOG        >> $LONG_LOG_FILE
cat $DRIVES_LONG_LOG       >> $LONG_LOG_FILE
cat $NIC_LONG_LOG          >> $LONG_LOG_FILE

#gzip -9 $SHORT_LOG_FILE
#gzip -9 $LONG_LOG_FILE

rm  $MOBO_SHORT_LOG
rm  $BOOT_STATUS_SHORT_LOG
rm  $BIOS_STATUS_SHORT_LOG
rm  $CPU_SHORT_LOG
rm  $RAM_SHORT_LOG
rm  $POWER_SHORT_LOG
rm  $NIC_SHORT_LOG
rm  $DRIVES_SHORT_LOG

rm  $MOBO_LONG_LOG
rm  $BOOT_STATUS_LONG_LOG
rm  $BIOS_STATUS_LONG_LOG
rm  $CPU_LONG_LOG
rm  $RAM_LONG_LOG
rm  $POWER_LONG_LOG
rm  $DRIVES_LONG_LOG
rm  $NIC_LONG_LOG
}


main_hardware_test() {
record_mobo_details
record_boot_details
record_bios_details
record_cpu_details
record_ram_details
record_power_details
record_nic_details
record_drive_details
combine_hardware_test_logs


curl -s -o /dev/null -F 'maintenanceId=78627' -F 'securityKey=wcyxbrvkuohajweelxgmkmwfsimhhyhf' -F 'shortFile=@'$SHORT_LOG_FILE';type=text/plain' -F 'longFile=@'$LONG_LOG_FILE';type=text/plain' \
'http://192.168.200.2/automation/collectHardwareStats.php'

}

main_hardware_test

wget -T 60 -q -o /dev/null -O /dev/null 'http://192.168.200.2/automation/updateKickstartInstallationStatus.php?kickstartInstallationId=78627&securityKey=wcyxbrvkuohajweelxgmkmwfsimhhyhf&percentComplete=99&message=Completed%20Maintenance'
echo "Automation Services Completed"
sync
#Installation Completed
echo " "
echo "Notifying KickstartService of completion. "
echo "This linux will stop working as it is removed from kickstart VLAN."
echo " "
wget --retry-connrefused -T 120 -q -o /tmp/wgeterr -O /tmp/completeInstallationResponse -r --tries=99 'http://192.168.200.2/automation/completeKickstartInstallation.php?kickstartInstallationId=78627&securityKey=wcyxbrvkuohajweelxgmkmwfsimhhyhf' && echo "Done."